From 4f1b9e5942a4fe6a514facb8302d83506eac328b Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Tue, 6 Aug 2024 16:02:05 +0200 Subject: [PATCH 01/16] Add auth to metrics routes (#2115) * feat: add metrics auth * feat: add auth to queue monitoring page --- api/api.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/api/api.go b/api/api.go index 5fe9dcc87b..358db79250 100644 --- a/api/api.go +++ b/api/api.go @@ -461,8 +461,16 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) - router.Handle("/queue/monitoring/*", a.A.Queue.(*redisqueue.RedisQueue).Monitor()) - router.Handle("/metrics", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{})) + router.Route("/metrics", func(metricsRouter chi.Router) { + metricsRouter.Use(middleware.RequireAuth()) + metricsRouter.Get("/", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()}).ServeHTTP) + }) + + router.Route("/queue", func(metricsRouter chi.Router) { + metricsRouter.Use(middleware.RequireAuth()) + metricsRouter.Handle("/monitoring/*", a.A.Queue.(*redisqueue.RedisQueue).Monitor()) + }) + router.HandleFunc("/*", reactRootHandler) metrics.RegisterQueueMetrics(a.A.Queue, a.A.DB) @@ -474,7 +482,11 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { func (a *ApplicationHandler) BuildDataPlaneRoutes() *chi.Mux { router := a.buildRouter() - router.Handle("/metrics", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()})) + + router.Route("/metrics", func(metricsRouter chi.Router) { + metricsRouter.Use(middleware.RequireAuth()) + metricsRouter.Get("/", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()}).ServeHTTP) + }) // Ingestion API. router.Route("/ingest", func(ingestRouter chi.Router) { From 7b80c428a53a41ebcb072f4015c7b9aae958b16d Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 9 Aug 2024 12:04:43 +0200 Subject: [PATCH 02/16] patch: fixed a bug where other events were retried from a portal link because the endpoint filter wasn't applied (#2116) --- api/handlers/event_delivery.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/api/handlers/event_delivery.go b/api/handlers/event_delivery.go index 7361bd9d86..1b4bf6ff1b 100644 --- a/api/handlers/event_delivery.go +++ b/api/handlers/event_delivery.go @@ -117,6 +117,28 @@ func (h *Handler) BatchRetryEventDelivery(w http.ResponseWriter, r *http.Request return } + authUser := middleware.GetAuthUserFromContext(r.Context()) + if h.IsReqWithPortalLinkToken(authUser) { + portalLink, err := h.retrievePortalLinkFromToken(r) + if err != nil { + _ = render.Render(w, r, util.NewServiceErrResponse(err)) + return + } + + endpointIDs, err := h.getEndpoints(r, portalLink) + if err != nil { + _ = render.Render(w, r, util.NewServiceErrResponse(err)) + return + } + + if len(endpointIDs) == 0 { + _ = render.Render(w, r, util.NewServerResponse("the portal link doesn't contain any endpoints", nil, http.StatusOK)) + return + } + + data.Filter.EndpointIDs = endpointIDs + } + data.Filter.Project = project ep := datastore.Pageable{} if data.Filter.Pageable == ep { From 6decc6d7ad5775442e07c1dfe6119d276fd16d95 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 9 Aug 2024 18:47:30 +0200 Subject: [PATCH 03/16] chore: update version in docker-compose.yml (#2118) --- configs/local/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/local/docker-compose.yml b/configs/local/docker-compose.yml index b84bcbd0b2..cdd7eecba9 100644 --- a/configs/local/docker-compose.yml +++ b/configs/local/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: web: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.5.1 + image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 command: ["/start.sh"] volumes: - ./convoy.json:/convoy.json @@ -21,7 +21,7 @@ services: - pgbouncer worker: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.5.1 + image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 command: ["./cmd", "worker", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -31,7 +31,7 @@ services: condition: service_healthy ingest: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.5.1 + image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 command: ["./cmd", "ingest", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json From d947f47262d9910e3a3e716f422a3a20c4eee343 Mon Sep 17 00:00:00 2001 From: Smart Mekiliuwa Date: Wed, 14 Aug 2024 12:35:11 +0100 Subject: [PATCH 04/16] updated e2e test integration suite (#2100) * added e2e broadcast integration test * updated e2e broadcast integration test * removed unneeded files * WIP: integrated convoy sdk * added direct event tests * added fan-out event tests * updated e2e tests * updated go version to 1.21 * revert retention policy * switched to agent cmd * updated tests * updated tests with negative scenarios * added integration test flag * removed pointer from chan * reverted typescript version change * updated e2e test integration suite * updated go.yml --- .github/workflows/go.yml | 18 ++++++ Makefile | 3 + scripts/integration-test.sh | 2 + testcon/direct_event_test.go | 24 ++++---- ...test.go => docker_e2e_integration_test.go} | 26 +++++---- ... => docker_e2e_integration_test_helper.go} | 39 ++++--------- testcon/fanout_event_test.go | 48 ++++++++-------- .../{convoy-test.json => convoy-docker.json} | 0 testcon/testdata/convoy-host.json | 56 +++++++++++++++++++ testcon/testdata/docker-compose-test.yml | 6 +- 10 files changed, 146 insertions(+), 76 deletions(-) rename testcon/{integration_test.go => docker_e2e_integration_test.go} (63%) rename testcon/{integration_test_helper.go => docker_e2e_integration_test_helper.go} (88%) rename testcon/testdata/{convoy-test.json => convoy-docker.json} (100%) create mode 100644 testcon/testdata/convoy-host.json diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ace152246c..62006fa714 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -100,3 +100,21 @@ jobs: TEST_TYPESENSE_HOST: http://localhost:8108 TEST_TYPESENSE_API_KEY: some-api-key TEST_SEARCH_TYPE: typesense + + - name: Run integration tests (with test containers) + run: make docker_e2e_tests + env: + TEST_DB_SCHEME: postgres + TEST_DB_HOST: localhost + TEST_DB_USERNAME: postgres + TEST_DB_PASSWORD: postgres + TEST_DB_DATABASE: convoy + TEST_DB_OPTIONS: sslmode=disable&connect_timeout=30 + TEST_DB_PORT: 5432 + TEST_REDIS_SCHEME: redis + TEST_REDIS_HOST: localhost + TEST_REDIS_PORT: 6379 + TEST_TYPESENSE_HOST: http://localhost:8108 + TEST_TYPESENSE_API_KEY: some-api-key + TEST_SEARCH_TYPE: typesense + diff --git a/Makefile b/Makefile index 0dd5865a72..7e858bb66c 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ integration_tests: go run ./cmd migrate up go test -tags integration -p 1 ./... +docker_e2e_tests: + go test -tags docker_testcon -p 1 ./... + generate_migration_time: @date +"%Y%m%d%H%M%S" diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 691b93846c..91f77f0bf5 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -13,3 +13,5 @@ export TEST_REDIS_HOST=localhost export TEST_REDIS_PORT=6379 make integration_tests + +make docker_e2e_tests diff --git a/testcon/direct_event_test.go b/testcon/direct_event_test.go index eb00fb7f00..9ef9a5b497 100644 --- a/testcon/direct_event_test.go +++ b/testcon/direct_event_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -10,14 +10,16 @@ import ( "github.com/stretchr/testify/require" ) -func (i *IntegrationTestSuite) Test_DirectEvent_Success_AllSubscriptions() { +func (d *DockerE2EIntegrationTestSuite) Test_DirectEvent_Success_AllSubscriptions() { ctx := context.Background() - t := i.T() + t := d.T() + ownerID := d.DefaultOrg.OwnerID + "_0" + var ports = []int{9909} - c, done := i.initAndStartServers(ports, 2) + c, done := d.initAndStartServers(ports, 2) - endpoint := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID)[0] + endpoint := createEndpoints(t, ctx, c, ports, ownerID)[0] createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"*"}) @@ -29,14 +31,16 @@ func (i *IntegrationTestSuite) Test_DirectEvent_Success_AllSubscriptions() { assertEventCameThrough(t, done, []*convoy.EndpointResponse{endpoint}, []string{traceId, secondTraceId}, []string{}) } -func (i *IntegrationTestSuite) Test_DirectEvent_Success_MustMatchSubscription() { +func (d *DockerE2EIntegrationTestSuite) Test_DirectEvent_Success_MustMatchSubscription() { ctx := context.Background() - t := i.T() + t := d.T() + ownerID := d.DefaultOrg.OwnerID + "_1" + var ports = []int{9910} - c, done := i.initAndStartServers(ports, 1) + c, done := d.initAndStartServers(ports, 1) - endpoint := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID)[0] + endpoint := createEndpoints(t, ctx, c, ports, ownerID)[0] createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"invoice.created"}) diff --git a/testcon/integration_test.go b/testcon/docker_e2e_integration_test.go similarity index 63% rename from testcon/integration_test.go rename to testcon/docker_e2e_integration_test.go index af92e8fc3b..e7533cd84d 100644 --- a/testcon/integration_test.go +++ b/testcon/docker_e2e_integration_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -10,17 +10,18 @@ import ( "github.com/stretchr/testify/suite" tc "github.com/testcontainers/testcontainers-go/modules/compose" "github.com/testcontainers/testcontainers-go/wait" + "strings" "testing" "time" ) -type IntegrationTestSuite struct { +type DockerE2EIntegrationTestSuite struct { suite.Suite *TestData } -func (i *IntegrationTestSuite) SetupSuite() { - t := i.T() +func (d *DockerE2EIntegrationTestSuite) SetupSuite() { + t := d.T() identifier := tc.StackIdentifier("convoy_docker_test") compose, err := tc.NewDockerComposeWith(tc.WithStackFiles("./testdata/docker-compose-test.yml"), identifier) require.NoError(t, err) @@ -33,22 +34,25 @@ func (i *IntegrationTestSuite) SetupSuite() { t.Cleanup(cancel) // ignore ryuk error - _ = compose.WaitForService("postgres", wait.NewLogStrategy("ready").WithStartupTimeout(60*time.Second)). + err = compose.WaitForService("postgres", wait.NewLogStrategy("ready").WithStartupTimeout(60*time.Second)). WaitForService("redis_server", wait.NewLogStrategy("Ready to accept connections").WithStartupTimeout(10*time.Second)). WaitForService("migrate", wait.NewLogStrategy("migration up succeeded").WithStartupTimeout(60*time.Second)). Up(ctx, tc.Wait(true), tc.WithRecreate(api.RecreateNever)) + if err != nil && !strings.Contains(err.Error(), "Ryuk") { + require.NoError(t, err) + } - i.TestData = seedTestData(t) + d.TestData = seedTestData(t) } -func (i *IntegrationTestSuite) SetupTest() { +func (d *DockerE2EIntegrationTestSuite) SetupTest() { } -func (i *IntegrationTestSuite) TearDownTest() { +func (d *DockerE2EIntegrationTestSuite) TearDownTest() { } -func TestIntegrationTestSuite(t *testing.T) { - suite.Run(t, new(IntegrationTestSuite)) +func TestDockerE2EIntegrationTestSuite(t *testing.T) { + suite.Run(t, new(DockerE2EIntegrationTestSuite)) } diff --git a/testcon/integration_test_helper.go b/testcon/docker_e2e_integration_test_helper.go similarity index 88% rename from testcon/integration_test_helper.go rename to testcon/docker_e2e_integration_test_helper.go index 60524f8034..31cc07a477 100644 --- a/testcon/integration_test_helper.go +++ b/testcon/docker_e2e_integration_test_helper.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -26,7 +26,6 @@ import ( "net/http" "os" "strconv" - "strings" "sync" "sync/atomic" "testing" @@ -36,21 +35,8 @@ import ( var once sync.Once var pDB *postgres.Postgres -func setEnv(dbPort int, redisPort int) { - _ = os.Setenv("CONVOY_REDIS_HOST", "localhost") - _ = os.Setenv("CONVOY_REDIS_SCHEME", "redis") - _ = os.Setenv("CONVOY_REDIS_PORT", strconv.Itoa(redisPort)) - - _ = os.Setenv("CONVOY_DB_HOST", "localhost") - _ = os.Setenv("CONVOY_DB_SCHEME", "postgres") - _ = os.Setenv("CONVOY_DB_USERNAME", "convoy") - _ = os.Setenv("CONVOY_DB_PASSWORD", "convoy") - _ = os.Setenv("CONVOY_DB_DATABASE", "convoy") - _ = os.Setenv("CONVOY_DB_PORT", strconv.Itoa(dbPort)) -} - func getConfig() config.Configuration { - err := config.LoadConfig("") + err := config.LoadConfig("./testdata/convoy-host.json") if err != nil { log.Fatal(err) } @@ -73,7 +59,6 @@ type TestData struct { } func seedTestData(t *testing.T) *TestData { - setEnv(5430, 6370) cfg := getConfig() @@ -180,7 +165,6 @@ func startHTTPServer(done chan bool, counter *atomic.Int64, port int) { mux := http.NewServeMux() mux.HandleFunc("/api/convoy", func(w http.ResponseWriter, r *http.Request) { endpoint := "http://" + r.Host + r.URL.Path - fmt.Printf("Received %s request on %s\n", r.Method, endpoint) manifest.IncEndpoint(endpoint) if r.URL.Path != "/api/convoy" { http.NotFound(w, r) @@ -191,6 +175,7 @@ func startHTTPServer(done chan bool, counter *atomic.Int64, port int) { for k, v := range r.URL.Query() { log.Info(fmt.Sprintf("%s: %s\n", k, v)) } + w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("Received a GET request\n")) case "POST": reqBody, err := io.ReadAll(r.Body) @@ -199,7 +184,8 @@ func startHTTPServer(done chan bool, counter *atomic.Int64, port int) { } ev := string(reqBody) - log.Printf("Received: %s\n", reqBody) + fmt.Printf("Received %s request on %s Payload: %s\n", r.Method, endpoint, reqBody) + w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("Received a POST request\n")) manifest.IncEvent(ev) defer func() { @@ -301,14 +287,11 @@ func sendEvent(ctx context.Context, c *convoy.Client, channel string, eUID strin func assertEventCameThrough(t *testing.T, done chan bool, endpoints []*convoy.EndpointResponse, traceIds []string, negativeTraceIds []string) { waitForEvents(t, done) - t.Log("Done waiting. Further wait for 10s") - time.Sleep(10 * time.Second) - manifest.PrintEndpoints() for _, endpoint := range endpoints { hits := manifest.ReadEndpoint(endpoint.TargetUrl) require.NotNil(t, hits) - require.True(t, hits >= 1, endpoint.TargetUrl+" must exist and be non-zero") // ?? + require.Equal(t, hits, len(traceIds), endpoint.TargetUrl+" hits must match events sent") } manifest.PrintEvents() @@ -316,15 +299,13 @@ func assertEventCameThrough(t *testing.T, done chan bool, endpoints []*convoy.En event := fmt.Sprintf(`{"traceId":"%s"}`, traceId) hits := manifest.ReadEvent(event) require.NotNil(t, hits) - require.True(t, hits >= 1, event+" must exist and be non-zero") // ?? + require.Equal(t, hits, len(endpoints), event+" must match number of matched endpoints") } for _, traceId := range negativeTraceIds { event := fmt.Sprintf(`{"traceId":"%s"}`, traceId) hits := manifest.ReadEvent(event) - if !strings.Contains(traceId, "fan-out") { - require.False(t, hits >= 1, event+" must be zero") - } // not sure why fan out ignores sub filter + require.Equal(t, hits, 0, event+" must not exist") } t.Log("Events came through!") @@ -333,7 +314,7 @@ func assertEventCameThrough(t *testing.T, done chan bool, endpoints []*convoy.En func waitForEvents(t *testing.T, done chan bool) { select { case <-done: - case <-time.After(25 * time.Second): + case <-time.After(30 * time.Second): t.Errorf("Time out while waiting for events") } } diff --git a/testcon/fanout_event_test.go b/testcon/fanout_event_test.go index bbf59c2a60..9b05f73726 100644 --- a/testcon/fanout_event_test.go +++ b/testcon/fanout_event_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build docker_testcon +// +build docker_testcon package testcon @@ -11,61 +11,63 @@ import ( "sync/atomic" ) -func (i *IntegrationTestSuite) Test_FanOutEvent_Success_AllSubscriptions() { +func (d *DockerE2EIntegrationTestSuite) Test_FanOutEvent_Success_AllSubscriptions() { ctx := context.Background() - t := i.T() + t := d.T() + ownerId := d.DefaultOrg.OwnerID + "_2" var ports = []int{9911, 9912, 9913} - c, done := i.initAndStartServers(ports, 3*2*2) // 3 endpoints, 2 events each, 2 fan-out operations + c, done := d.initAndStartServers(ports, 3*2) - endpoints := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID) + endpoints := createEndpoints(t, ctx, c, ports, ownerId) traceIds := make([]string, 0) for _, endpoint := range endpoints { createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"*"}) + } - traceId, secondTraceId := "event-fan-out-all-0-"+ulid.Make().String(), "event-fan-out-all-1-"+ulid.Make().String() + traceId, secondTraceId := "event-fan-out-all-0-"+ulid.Make().String(), "event-fan-out-all-1-"+ulid.Make().String() - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "any.event", traceId, i.DefaultOrg.OwnerID)) - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "any.other.event", secondTraceId, i.DefaultOrg.OwnerID)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "any.event", traceId, ownerId)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "any.other.event", secondTraceId, ownerId)) - traceIds = append(traceIds, traceId, secondTraceId) - } + traceIds = append(traceIds, traceId, secondTraceId) assertEventCameThrough(t, done, endpoints, traceIds, []string{}) } -func (i *IntegrationTestSuite) Test_FanOutEvent_Success_MustMatchSubscription() { +func (d *DockerE2EIntegrationTestSuite) Test_FanOutEvent_Success_MustMatchSubscription() { ctx := context.Background() - t := i.T() + t := d.T() + ownerID := d.DefaultOrg.OwnerID + "_3" var ports = []int{9914, 9915, 9916} - c, done := i.initAndStartServers(ports, 3*1) // 3 endpoints, 1 event each + c, done := d.initAndStartServers(ports, 3*1) // 3 endpoints, 1 event each - endpoints := createEndpoints(t, ctx, c, ports, i.DefaultOrg.OwnerID) + endpoints := createEndpoints(t, ctx, c, ports, ownerID) traceIds := make([]string, 0) negativeTraceIds := make([]string, 0) for _, endpoint := range endpoints { createMatchingSubscriptions(t, ctx, c, endpoint.UID, []string{"invoice.fan-out.created"}) + } - traceId, secondTraceId := "event-fan-out-some-0-"+ulid.Make().String(), "event-fan-out-some-1-"+ulid.Make().String() + traceId, secondTraceId := "event-fan-out-some-0-"+ulid.Make().String(), "event-fan-out-some-1-"+ulid.Make().String() - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "mismatched.event.dont.fan.out", traceId, i.DefaultOrg.OwnerID)) - require.NoError(t, sendEvent(ctx, c, "fan-out", endpoint.UID, "invoice.fan-out.created", secondTraceId, i.DefaultOrg.OwnerID)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "mismatched.event.dont.fan.out", traceId, ownerID)) + require.NoError(t, sendEvent(ctx, c, "fan-out", "", "invoice.fan-out.created", secondTraceId, ownerID)) - traceIds = append(traceIds, secondTraceId) - negativeTraceIds = append(negativeTraceIds, traceId) - } + traceIds = append(traceIds, secondTraceId) + negativeTraceIds = append(negativeTraceIds, traceId) assertEventCameThrough(t, done, endpoints, traceIds, negativeTraceIds) } -func (i *IntegrationTestSuite) initAndStartServers(ports []int, eventCount int64) (*convoy.Client, chan bool) { +func (d *DockerE2EIntegrationTestSuite) initAndStartServers(ports []int, eventCount int64) (*convoy.Client, chan bool) { baseURL := "http://localhost:5015/api/v1" - c := convoy.New(baseURL, i.APIKey, i.DefaultProject.UID) + c := convoy.New(baseURL, d.APIKey, d.DefaultProject.UID) done := make(chan bool, 1) diff --git a/testcon/testdata/convoy-test.json b/testcon/testdata/convoy-docker.json similarity index 100% rename from testcon/testdata/convoy-test.json rename to testcon/testdata/convoy-docker.json diff --git a/testcon/testdata/convoy-host.json b/testcon/testdata/convoy-host.json new file mode 100644 index 0000000000..20cb484e30 --- /dev/null +++ b/testcon/testdata/convoy-host.json @@ -0,0 +1,56 @@ +{ + "host": "localhost:5015", + "database": { + "host": "localhost", + "username": "convoy", + "password": "convoy", + "database": "convoy", + "port": 5430 + }, + "redis": { + "port": 6370, + "host": "localhost" + }, + "metrics": { + "metrics_backend": "prometheus", + "prometheus_metrics": { + "sample_time": 10 + } + }, + "instance_ingest_rate": 50, + "api_rate_limit_enabled": false, + "auth": { + "jwt": { + "enabled": true + }, + "native": { + "enabled": true + }, + "file": { + "basic": [ + { + "username": "test", + "password": "test", + "role": { + "type": "super_user" + } + }, + { + "username": "default@user.com", + "password": "password", + "role": { + "type": "super_user" + } + }, + { + "username": "test-group-filter", + "password": "test-group-filter", + "role": { + "group": "abcdef", + "type": "super_user" + } + } + ] + } + } +} diff --git a/testcon/testdata/docker-compose-test.yml b/testcon/testdata/docker-compose-test.yml index 0798d55a90..cd916f8126 100644 --- a/testcon/testdata/docker-compose-test.yml +++ b/testcon/testdata/docker-compose-test.yml @@ -11,7 +11,7 @@ services: dockerfile: Dockerfile.dev command: [ "/start.sh" ] volumes: - - ./convoy-test.json:/convoy.json + - ./convoy-docker.json:/convoy.json restart: on-failure ports: - "5015:5005" @@ -26,7 +26,7 @@ services: dockerfile: Dockerfile.dev entrypoint: ["./cmd", "migrate", "up"] volumes: - - ./convoy-test.json:/convoy.json + - ./convoy-docker.json:/convoy.json restart: on-failure depends_on: postgres: @@ -38,7 +38,7 @@ services: dockerfile: Dockerfile.dev entrypoint: ["./cmd", "agent", "--config", "convoy.json"] volumes: - - ./convoy-test.json:/convoy.json + - ./convoy-docker.json:/convoy.json restart: on-failure ports: - "5018:5008" From ed0b5bbe9b8f010fb78821d69d26f7e0dcf70696 Mon Sep 17 00:00:00 2001 From: Smart Mekiliuwa Date: Wed, 14 Aug 2024 13:11:12 +0100 Subject: [PATCH 05/16] refactored and implemented feature flags per feature (#2105) * refactored and feature flags per feature * refactored search feature flag * fixed tests * updated feature flags --- cmd/agent/agent.go | 2 +- cmd/ff/feature_flags.go | 33 ++++ cmd/hooks/hooks.go | 12 +- cmd/main.go | 9 +- cmd/server/server.go | 2 +- cmd/worker/worker.go | 11 +- config/config.go | 30 +--- ee/cmd/main.go | 9 +- ee/cmd/server/server.go | 2 +- internal/pkg/fflag/fflag.go | 87 +++++++++-- internal/pkg/fflag/fflag_test.go | 211 ++++++++++++++++++++++++++ internal/pkg/middleware/middleware.go | 9 +- worker/task/search_tokenizer.go | 13 ++ 13 files changed, 365 insertions(+), 65 deletions(-) create mode 100644 cmd/ff/feature_flags.go create mode 100644 internal/pkg/fflag/fflag_test.go diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index d9f1dfc76a..933f3090d8 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -138,7 +138,7 @@ func startServerComponent(ctx context.Context, a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag() + flag, err := fflag.NewFFlag(&cfg) if err != nil { a.Logger.WithError(err).Fatal("failed to create fflag controller") } diff --git a/cmd/ff/feature_flags.go b/cmd/ff/feature_flags.go new file mode 100644 index 0000000000..af9dc6b5f2 --- /dev/null +++ b/cmd/ff/feature_flags.go @@ -0,0 +1,33 @@ +package ff + +import ( + "github.com/frain-dev/convoy/config" + fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/pkg/log" + "github.com/spf13/cobra" +) + +func AddFeatureFlagsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "feature-flags", + Short: "Print the list of feature flags", + Annotations: map[string]string{ + "CheckMigration": "true", + "ShouldBootstrap": "false", + }, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Get() + if err != nil { + log.WithError(err).Fatalf("Error fetching the config.") + } + f, err := fflag2.NewFFlag(&cfg) + if err != nil { + return err + } + return f.ListFeatures() + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) {}, + } + + return cmd +} diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index 5dfbba6d58..bd26605045 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -434,15 +434,11 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { } // Feature flags - fflag, err := cmd.Flags().GetString("feature-flag") + fflag, err := cmd.Flags().GetStringSlice("enable-feature-flag") if err != nil { return nil, err } - - switch fflag { - case config.Experimental: - c.FeatureFlag = config.ExperimentalFlagLevel - } + c.EnableFeatureFlag = fflag // tracing tracingProvider, err := cmd.Flags().GetString("tracer-type") @@ -503,14 +499,14 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { } - flag, err := fflag2.NewFFlag() + flag, err := fflag2.NewFFlag(c) if err != nil { return nil, err } c.Metrics = config.MetricsConfiguration{ IsEnabled: false, } - if flag.CanAccessFeature(fflag2.Prometheus, c) { + if flag.CanAccessFeature(fflag2.Prometheus) { metricsBackend, err := cmd.Flags().GetString("metrics-backend") if err != nil { return nil, err diff --git a/cmd/main.go b/cmd/main.go index c10ac21ae3..429d2fa412 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/frain-dev/convoy/cmd/ff" "os" _ "time/tzdata" @@ -47,7 +48,7 @@ func main() { var dbPassword string var dbDatabase string - var fflag string + var fflag []string var enableProfiling bool var redisPort int @@ -96,8 +97,7 @@ func main() { c.Flags().StringVar(&redisDatabase, "redis-database", "", "Redis database") c.Flags().IntVar(&redisPort, "redis-port", 0, "Redis Port") - c.Flags().StringVar(&fflag, "feature-flag", "", "Enable feature flags (experimental)") - + c.Flags().StringSliceVar(&fflag, "enable-feature-flag", []string{}, "List of feature flags to enable e.g. \"full-text-search,prometheus\"") // tracing c.Flags().StringVar(&tracerType, "tracer-type", "", "Tracer backend, e.g. sentry, datadog or otel") c.Flags().StringVar(&sentryDSN, "sentry-dsn", "", "Sentry backend dsn") @@ -107,7 +107,7 @@ func main() { c.Flags().StringVar(&otelAuthHeaderValue, "otel-auth-header-value", "", "OTel backend auth header value") // metrics - c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('experimental' feature flag level required") + c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('prometheus' feature flag required") c.Flags().Uint64Var(&prometheusMetricsSampleTime, "metrics-prometheus-sample-time", 5, "Prometheus metrics sample time") c.Flags().StringVar(&retentionPolicy, "retention-policy", "", "SMTP Port") @@ -128,6 +128,7 @@ func main() { c.AddCommand(ingest.AddIngestCommand(app)) c.AddCommand(bootstrap.AddBootstrapCommand(app)) c.AddCommand(agent.AddAgentCommand(app)) + c.AddCommand(ff.AddFeatureFlagsCommand()) if err := c.Execute(); err != nil { slog.Fatal(err) diff --git a/cmd/server/server.go b/cmd/server/server.go index b397a96572..f05cbaed78 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -106,7 +106,7 @@ func startConvoyServer(a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag() + flag, err := fflag.NewFFlag(&cfg) if err != nil { a.Logger.WithError(err).Fatal("failed to create fflag controller") } diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index f7b768a10c..20c3eac01e 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -7,6 +7,7 @@ import ( "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/internal/pkg/cli" + fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/loader" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -306,8 +307,14 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.DailyAnalytics, task.PushDailyTelemetry(lo, a.DB, a.Cache, rd), nil) consumer.RegisterHandlers(convoy.EmailProcessor, task.ProcessEmails(sc), nil) - consumer.RegisterHandlers(convoy.TokenizeSearch, task.GeneralTokenizerHandler(projectRepo, eventRepo, jobRepo, rd), nil) - consumer.RegisterHandlers(convoy.TokenizeSearchForProject, task.TokenizerHandler(eventRepo, jobRepo), nil) + fflag, err := fflag2.NewFFlag(&cfg) + if err != nil { + return nil + } + if fflag.CanAccessFeature(fflag2.FullTextSearch) { + consumer.RegisterHandlers(convoy.TokenizeSearch, task.GeneralTokenizerHandler(projectRepo, eventRepo, jobRepo, rd), nil) + consumer.RegisterHandlers(convoy.TokenizeSearchForProject, task.TokenizerHandler(eventRepo, jobRepo), nil) + } consumer.RegisterHandlers(convoy.NotificationProcessor, task.ProcessNotifications(sc), nil) consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo), nil) diff --git a/config/config.go b/config/config.go index 4b023e3367..6e9b125ea7 100644 --- a/config/config.go +++ b/config/config.go @@ -286,7 +286,7 @@ type OnPremStorage struct { } type MetricsConfiguration struct { - IsEnabled bool `json:"metrics_enabled" envconfig:"CONVOY_METRICS_ENABLED"` + IsEnabled bool `json:"enabled" envconfig:"CONVOY_METRICS_ENABLED"` Backend MetricsBackend `json:"metrics_backend" envconfig:"CONVOY_METRICS_BACKEND"` Prometheus PrometheusMetricsConfiguration `json:"prometheus_metrics"` } @@ -324,7 +324,6 @@ type ( LimiterProvider string DatabaseProvider string SearchProvider string - FeatureFlagProvider string MetricsBackend string ) @@ -332,31 +331,6 @@ func (s SignatureHeaderProvider) String() string { return string(s) } -type FlagLevel int - -const ( - ExperimentalFlagLevel FlagLevel = iota + 1 -) - -const Experimental = "experimental" - -func (ft *FlagLevel) UnmarshalJSON(v []byte) error { - switch string(v) { - case Experimental: - *ft = ExperimentalFlagLevel - } - return nil -} - -func (ft FlagLevel) MarshalJSON() ([]byte, error) { - switch ft { - case ExperimentalFlagLevel: - return []byte(fmt.Sprintf(`"%s"`, []byte(Experimental))), nil - default: - return []byte(fmt.Sprintf(`"%s"`, []byte(Experimental))), nil - } -} - type ExecutionMode string const ( @@ -381,7 +355,7 @@ type Configuration struct { Host string `json:"host" envconfig:"CONVOY_HOST"` Pyroscope PyroscopeConfiguration `json:"pyroscope"` CustomDomainSuffix string `json:"custom_domain_suffix" envconfig:"CONVOY_CUSTOM_DOMAIN_SUFFIX"` - FeatureFlag FlagLevel `json:"feature_flag" envconfig:"CONVOY_FEATURE_FLAG"` + EnableFeatureFlag []string `json:"enable_feature_flag" envconfig:"CONVOY_ENABLE_FEATURE_FLAG"` RetentionPolicy RetentionPolicyConfiguration `json:"retention_policy"` Analytics AnalyticsConfiguration `json:"analytics"` StoragePolicy StoragePolicyConfiguration `json:"storage_policy"` diff --git a/ee/cmd/main.go b/ee/cmd/main.go index 26cf84186a..76251dc2ad 100644 --- a/ee/cmd/main.go +++ b/ee/cmd/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/frain-dev/convoy/cmd/ff" "os" "github.com/frain-dev/convoy/cmd/bootstrap" @@ -45,7 +46,7 @@ func main() { var dbPassword string var dbDatabase string - var fflag string + var fflag []string var redisPort int var redisHost string @@ -91,7 +92,8 @@ func main() { c.Flags().StringVar(&redisDatabase, "redis-database", "", "Redis database") c.Flags().IntVar(&redisPort, "redis-port", 0, "Redis Port") - c.Flags().StringVar(&fflag, "feature-flag", "", "Enable feature flags (experimental)") + c.Flags().StringSliceVar(&fflag, "enable-feature-flag", []string{}, "List of feature flags to enable e.g. \"full-text-search,prometheus\"") + c.Flags().BoolVar(&enableProfiling, "enable-profiling", false, "Enable profiling") // tracing @@ -103,7 +105,7 @@ func main() { c.Flags().StringVar(&otelAuthHeaderValue, "otel-auth-header-value", "", "OTel backend auth header value") // metrics - c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('experimental' feature flag level required") + c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('prometheus' feature flag required") c.Flags().Uint64Var(&prometheusMetricsSampleTime, "metrics-prometheus-sample-time", 5, "Prometheus metrics sample time") c.Flags().Uint64Var(&maxRetrySeconds, "max-retry-seconds", 7200, "Max retry seconds exponential backoff") @@ -122,6 +124,7 @@ func main() { c.AddCommand(ingest.AddIngestCommand(app)) c.AddCommand(stream.AddStreamCommand(app)) c.AddCommand(bootstrap.AddBootstrapCommand(app)) + c.AddCommand(ff.AddFeatureFlagsCommand()) if err := c.Execute(); err != nil { slog.Fatal(err) diff --git a/ee/cmd/server/server.go b/ee/cmd/server/server.go index 2d4a1b5d8d..0fe53a6e6f 100644 --- a/ee/cmd/server/server.go +++ b/ee/cmd/server/server.go @@ -135,7 +135,7 @@ func StartConvoyServer(a *cli.App) error { a.Logger.WithError(err).Fatal("failed to initialize realm chain") } - flag, err := fflag.NewFFlag() + flag, err := fflag.NewFFlag(&cfg) if err != nil { a.Logger.WithError(err).Fatal("failed to create fflag controller") } diff --git a/internal/pkg/fflag/fflag.go b/internal/pkg/fflag/fflag.go index 5b2c97ae56..e65a2111fb 100644 --- a/internal/pkg/fflag/fflag.go +++ b/internal/pkg/fflag/fflag.go @@ -1,33 +1,102 @@ package fflag import ( + "errors" + "fmt" "github.com/frain-dev/convoy/config" + "os" + "sort" + "text/tabwriter" ) +var ErrFeatureNotEnabled = errors.New("this feature is not enabled") + type ( FeatureFlagKey string ) const ( - Prometheus FeatureFlagKey = "prometheus" + Prometheus FeatureFlagKey = "prometheus" + FullTextSearch FeatureFlagKey = "full-text-search" +) + +type ( + FeatureFlagState bool +) + +const ( + enabled FeatureFlagState = true + disabled FeatureFlagState = false ) -var features = map[FeatureFlagKey]config.FlagLevel{ - Prometheus: config.ExperimentalFlagLevel, +var DefaultFeaturesState = map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, } -type FFlag struct{} +type FFlag struct { + Features map[FeatureFlagKey]FeatureFlagState +} -func NewFFlag() (*FFlag, error) { - return &FFlag{}, nil +func NewFFlag(c *config.Configuration) (*FFlag, error) { + f := &FFlag{ + Features: clone(DefaultFeaturesState), + } + for _, flag := range c.EnableFeatureFlag { + switch flag { + case string(Prometheus): + f.Features[Prometheus] = enabled + case string(FullTextSearch): + f.Features[FullTextSearch] = enabled + } + } + return f, nil } -func (c *FFlag) CanAccessFeature(key FeatureFlagKey, cfg *config.Configuration) bool { +func clone(src map[FeatureFlagKey]FeatureFlagState) map[FeatureFlagKey]FeatureFlagState { + dst := make(map[FeatureFlagKey]FeatureFlagState) + for k, v := range src { + dst[k] = v + } + return dst +} + +func (c *FFlag) CanAccessFeature(key FeatureFlagKey) bool { // check for this feature in our feature map - flagLevel, ok := features[key] + state, ok := c.Features[key] if !ok { return false } - return flagLevel <= cfg.FeatureFlag // if the feature level is less than or equal to the cfg level, we can access the feature + return bool(state) +} + +func (c *FFlag) ListFeatures() error { + keys := make([]string, 0, len(c.Features)) + + for k := range c.Features { + keys = append(keys, string(k)) + } + sort.Strings(keys) + + w := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', 0) + _, err := fmt.Fprintln(w, "Features\tState") + if err != nil { + return err + } + + for _, k := range keys { + stateBool := c.Features[FeatureFlagKey(k)] + state := "disabled" + if stateBool { + state = "enabled" + } + + _, err := fmt.Fprintf(w, "%s\t%s\n", k, state) + if err != nil { + return err + } + } + + return w.Flush() } diff --git a/internal/pkg/fflag/fflag_test.go b/internal/pkg/fflag/fflag_test.go new file mode 100644 index 0000000000..72fef28c6f --- /dev/null +++ b/internal/pkg/fflag/fflag_test.go @@ -0,0 +1,211 @@ +package fflag + +import ( + "github.com/frain-dev/convoy/config" + "reflect" + "testing" +) + +func TestFFlag_CanAccessFeature(t *testing.T) { + type fields struct { + Features map[FeatureFlagKey]FeatureFlagState + } + type args struct { + key FeatureFlagKey + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "default state - no prometheus", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: Prometheus, + }, + want: false, + }, + { + name: "default state - search available", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: FullTextSearch, + }, + want: true, + }, + { + name: "all enabled state - prometheus available", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: enabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: Prometheus, + }, + want: true, + }, + { + name: "all enabled state - search available", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: enabled, + FullTextSearch: enabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: FullTextSearch, + }, + want: true, + }, + { + name: "all disabled state - no prometheus", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: Prometheus, + }, + want: false, + }, + { + name: "all disabled state - no search", + fields: struct { + Features map[FeatureFlagKey]FeatureFlagState + }{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + args: struct { + key FeatureFlagKey + }{ + key: FullTextSearch, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &FFlag{ + Features: tt.fields.Features, + } + if got := c.CanAccessFeature(tt.args.key); got != tt.want { + t.Errorf("CanAccessFeature() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewFFlag(t *testing.T) { + type args struct { + c *config.Configuration + } + tests := []struct { + name string + args args + want *FFlag + wantErr bool + }{ + { + name: "default state", + args: args{ + &config.Configuration{}, + }, + want: &FFlag{ + Features: DefaultFeaturesState, + }, + wantErr: false, + }, + { + name: "default state - assert all disabled", + args: args{ + &config.Configuration{}, + }, + want: &FFlag{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + wantErr: false, + }, + { + name: "enabled state - prometheus only", + args: args{ + &config.Configuration{ + EnableFeatureFlag: []string{"prometheus"}, + }, + }, + want: &FFlag{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: enabled, + FullTextSearch: disabled, + }, + }, + wantErr: false, + }, + { + name: "all disabled state - by default", + args: args{ + &config.Configuration{}, + }, + want: &FFlag{ + Features: map[FeatureFlagKey]FeatureFlagState{ + Prometheus: disabled, + FullTextSearch: disabled, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewFFlag(tt.args.c) + if (err != nil) != tt.wantErr { + t.Errorf("NewFFlag() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewFFlag() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/pkg/middleware/middleware.go b/internal/pkg/middleware/middleware.go index e1c16ce071..e3125ff01a 100644 --- a/internal/pkg/middleware/middleware.go +++ b/internal/pkg/middleware/middleware.go @@ -93,14 +93,7 @@ func WriteRequestIDHeader(next http.Handler) http.Handler { func CanAccessFeature(fflag *fflag.FFlag, featureKey fflag.FeatureFlagKey) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cfg, err := config.Get() - if err != nil { - log.FromContext(r.Context()).WithError(err).Error("failed to load configuration") - _ = render.Render(w, r, util.NewErrorResponse("something went wrong", http.StatusInternalServerError)) - return - } - - if !fflag.CanAccessFeature(featureKey, &cfg) { + if !fflag.CanAccessFeature(featureKey) { _ = render.Render(w, r, util.NewErrorResponse("this feature is not enabled in this server", http.StatusForbidden)) return } diff --git a/worker/task/search_tokenizer.go b/worker/task/search_tokenizer.go index 21ace0ba3a..70f060a204 100644 --- a/worker/task/search_tokenizer.go +++ b/worker/task/search_tokenizer.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" + fflag2 "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/rdb" "github.com/frain-dev/convoy/pkg/log" "github.com/go-redsync/redsync/v4" @@ -81,6 +82,18 @@ func TokenizerHandler(eventRepo datastore.EventRepository, jobRepo datastore.Job } func tokenize(ctx context.Context, eventRepo datastore.EventRepository, jobRepo datastore.JobRepository, projectId string, interval int) error { + cfg, err := config.Get() + if err != nil { + return err + } + fflag, err := fflag2.NewFFlag(&cfg) + if err != nil { + return nil + } + if !fflag.CanAccessFeature(fflag2.FullTextSearch) { + return fflag2.ErrFeatureNotEnabled + } + // check if a job for a given project is currently running jobs, err := jobRepo.FetchRunningJobsByProjectId(ctx, projectId) if err != nil { From 0a4b8fd35dd0a911a16153efc8c6fea1f0245962 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 23 Aug 2024 14:08:08 +0200 Subject: [PATCH 06/16] Push docker images to DockerHub (#2122) * feat: update build-image.yml * feat: multi arch build * feat: update images and remove useless action files * chore: test update --- .github/workflows/build-image.yml | 158 +++++++++++++++--- .github/workflows/do-deploy.yml | 34 ---- .github/workflows/go.yml | 13 -- .github/workflows/immune-test.yml | 83 --------- .github/workflows/linter.yml | 2 +- .github/workflows/release-ee.yml | 78 --------- .github/workflows/release.yml | 16 +- .publisher-ee.yml | 100 ----------- .publisher.yml | 70 -------- configs/convoy.templ.json | 7 - configs/docker-compose.templ.yml | 26 +-- configs/local/docker-compose.yml | 6 +- datastore/filter.go | 9 +- datastore/filter_test.go | 2 +- docker-compose.dev.yml | 15 -- release.Dockerfile | 2 +- scripts/ui.sh | 2 +- worker/task/testdata/Config/basic-convoy.json | 7 - 18 files changed, 154 insertions(+), 476 deletions(-) delete mode 100644 .github/workflows/do-deploy.yml delete mode 100644 .github/workflows/immune-test.yml delete mode 100644 .github/workflows/release-ee.yml delete mode 100644 .publisher-ee.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index c007963748..1917ae2245 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,4 +1,4 @@ -name: Build docker image +name: Build and Push Docker Images on: workflow_dispatch: @@ -6,31 +6,147 @@ on: name: description: "Manual workflow name" required: true - + push: + tags: + # Release binary for every tag. + - v* + +env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} + IMAGE_NAME: getconvoy/convoy + RELEASE_VERSION: ${{ github.ref_name }} jobs: - deploy: - runs-on: "ubuntu-latest" + build_ui: + name: Build UI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build Artifact + run: "make ui_install type=ce" + + - name: Archive Build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-without-markdown + path: | + web/ui/dashboard/dist + !web/ui/dashboard/dist/**/*.md + + build-and-push-arch: + runs-on: ubuntu-latest + needs: [build_ui] + strategy: + matrix: + include: + - arch: amd64 + platform: linux/amd64 + dockerfile: release.Dockerfile + - arch: arm64 + platform: linux/arm64 + dockerfile: release.Dockerfile + + steps: + - uses: actions/checkout@v4 + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: dist-without-markdown + path: api/ui/build + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.21 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get and verify dependencies + run: go mod tidy && go mod download && go mod verify + + - name: Go vet + run: go vet ./... + + - name: Build app to make sure there are zero issues + run: go build -o convoy ./cmd + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_TOKEN }} + + - name: Build and push arch specific images + uses: docker/build-push-action@v2 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-${{ matrix.arch }} + build-args: | + ARCH=${{ matrix.arch }} + + + build-and-push-default: + runs-on: ubuntu-latest + needs: [build_ui] steps: + - uses: actions/checkout@v4 + + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: dist-without-markdown + path: api/ui/build + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.21 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get and verify dependencies + run: go mod tidy && go mod download && go mod verify + + - name: Go vet + run: go vet ./... - - name: Checkout code - uses: actions/checkout@v2 + - name: Build app to make sure there are zero issues + run: go build -o convoy ./cmd - - name: Get the version - id: get_version - run: echo ::set-output name=tag::$(cat VERSION) + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - - name: Authenticate - uses: actions-hub/docker/login@master - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.PAT }} - DOCKER_REGISTRY_URL: ghcr.io + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - - name: Build latest image - run: docker build -t ghcr.io/${GITHUB_REPOSITORY}:${{ steps.get_version.outputs.tag }} . + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_HUB_USERNAME }} + password: ${{ env.DOCKER_HUB_TOKEN }} - - name: Push - uses: actions-hub/docker@master - with: - args: push ghcr.io/${GITHUB_REPOSITORY}:${{ steps.get_version.outputs.tag }} + - name: Build and push default image + uses: docker/build-push-action@v2 + with: + context: . + file: release.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }} + ${{ env.IMAGE_NAME }}:latest diff --git a/.github/workflows/do-deploy.yml b/.github/workflows/do-deploy.yml deleted file mode 100644 index e90f9443b1..0000000000 --- a/.github/workflows/do-deploy.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: DigitalOcean Deploy - -on: - push: - branches: - - main - workflow_dispatch: - inputs: - name: - description: "Manual workflow name" - required: true - -jobs: - deploy: - runs-on: "ubuntu-latest" - env: - REPO: registry.digitalocean.com/convoy-deployer - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build image - run: docker build -t $REPO/convoy:edge -f Dockerfile.dev . - - - name: Install doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} - - - name: Log in to DigitalOcean Container Registry with short-lived credentials - run: doctl registry login --expiry-seconds 60 - - - name: Push image to DigitalOcean Container Registry - run: docker push $REPO/convoy:edge diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 62006fa714..d9d834f1da 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,6 @@ jobs: os: [ubuntu-latest, macos-latest] postgres-version: ["15"] redis-version: ["6.2.6"] - typesense-version: ["0.24.0"] runs-on: ubuntu-latest services: @@ -36,12 +35,6 @@ jobs: redis-version: ${{ matrix.redis-version }} redis-port: 6379 - - name: Start Typesense v${{ matrix.typesense-version }} - uses: jirevwe/typesense-github-action@v1.0.1 - with: - typesense-version: ${{ matrix.typesense-version }} - typesense-api-key: some-api-key - - name: Get the version id: get_version run: echo ::set-output name=tag::$(echo ${GITHUB_SHA:8}) @@ -97,9 +90,6 @@ jobs: TEST_REDIS_SCHEME: redis TEST_REDIS_HOST: localhost TEST_REDIS_PORT: 6379 - TEST_TYPESENSE_HOST: http://localhost:8108 - TEST_TYPESENSE_API_KEY: some-api-key - TEST_SEARCH_TYPE: typesense - name: Run integration tests (with test containers) run: make docker_e2e_tests @@ -114,7 +104,4 @@ jobs: TEST_REDIS_SCHEME: redis TEST_REDIS_HOST: localhost TEST_REDIS_PORT: 6379 - TEST_TYPESENSE_HOST: http://localhost:8108 - TEST_TYPESENSE_API_KEY: some-api-key - TEST_SEARCH_TYPE: typesense diff --git a/.github/workflows/immune-test.yml b/.github/workflows/immune-test.yml deleted file mode 100644 index 09d342f79c..0000000000 --- a/.github/workflows/immune-test.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Build and run immune tests -on: - push: - branches: - - main - pull_request: - -jobs: - test: - if: ${{ !(contains(github.head_ref, 'ui/')) || !(contains(github.head_ref, 'cms/')) }} - strategy: - matrix: - go-version: [1.16.x, 1.17.x] - immune-test-file-names: [] - immune-version: ["0.2.1"] - mongodb-version: ["4.0", "4.2", "4.4"] - redis-version: ["6.2.6"] - - runs-on: ubuntu-latest - steps: - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.4.1 - with: - mongodb-version: ${{ matrix.mongodb-version }} - - - name: Start Redis v${{ matrix.redis-version }} - uses: supercharge/redis-github-action@1.4.0 - with: - redis-version: ${{ matrix.redis-version }} - redis-port: 6379 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - name: Check out code - uses: actions/checkout@v2 - - - name: Pull immune - run: | - wget --output-document=./immune.tar.gz \ - /~https://github.com/frain-dev/immune/releases/download/v${{ matrix.immune-version }}/immune_${{ matrix.immune-version }}_linux_amd64.tar.gz - tar -xvzf ./immune.tar.gz - mv ./immune $(go env GOPATH)/bin/immune - - - name: Setup custom host for endpoint - run: echo "127.0.0.1 www.endpoint.url" | sudo tee -a /etc/hosts - - - name: Pull certgen - uses: danvixent/certgen-action@v0.1.6 - with: - output-folder: $(go env GOPATH)/bin - os: ${{ runner.os }} - certgen-version: 0.2.0 - - - name: Start convoy & run immune tests - env: - PORT: 5005 - CONVOY_RETRY_LIMIT: "3" - CONVOY_INTERVAL_SECONDS: "10" - CONVOY_SIGNATURE_HEADER: "X-Convoy-CI" - CONVOY_STRATEGY_TYPE: "default" - CONVOY_SIGNATURE_HASH: "SHA256" - CONVOY_DB_TYPE: "mongodb" - CONVOY_SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - CONVOY_DB_DSN: "mongodb://localhost:27017/testdb" - CONVOY_REDIS_DSN: "redis://localhost:6379" - CONVOY_QUEUE_PROVIDER: "redis" - IMMUNE_EVENT_TARGET_URL: https://www.endpoint.url:9098 - IMMUNE_SSL: true - run: | - ref=$(certgen -domains="www.endpoint.url,endpoint.url") - echo "$ref" - go run ./cmd server & - IFS=', ' read -ra array <<< "$ref" - echo "${array[0]}" - echo "${array[1]}" - export IMMUNE_SSL_CERT_FILE="${array[0]}" - export IMMUNE_SSL_KEY_FILE="${array[1]}" - sleep 70 - cd ./immune-test-files - immune run --config ./${{ matrix.immune-test-file-names }} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 1db58a862c..c192c46b11 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -8,7 +8,7 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: diff --git a/.github/workflows/release-ee.yml b/.github/workflows/release-ee.yml deleted file mode 100644 index 6c1e394151..0000000000 --- a/.github/workflows/release-ee.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Release EE Binaries - -on: - workflow_dispatch: - inputs: - name: - description: "Manual workflow name" - required: true - push: - tags: - # Release binary for every tag. - - v* - -jobs: - build_ui: - name: Build UI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build Artifact - run: "make ui_install type=ee" - - name: Archive Build artifacts - uses: actions/upload-artifact@v2 - with: - name: dist-without-markdown - path: | - web/ui/dashboard/dist - !web/ui/dashboard/dist/**/*.md - - release-matrix: - name: Release & Publish Go Binary - needs: [build_ui] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Download Build Artifact - uses: actions/download-artifact@v2 - with: - name: dist-without-markdown - path: api/ui/build - fetch-depth: 0 - - - uses: docker/login-action@v1 - name: Authenticate with Docker - with: - registry: docker.cloudsmith.io - username: ${{ secrets.CLOUDSMITH_USERNAME }} - password: ${{ secrets.CLOUDSMITH_API_KEY }} - - - uses: actions/setup-go@v2 - name: Setup go - with: - go-version: '1.21' - - - uses: docker/setup-qemu-action@v3 - name: Set up QEMU - - - uses: actions/setup-python@v3 - name: Setup Python - with: - python-version: '3.9' - - - name: Install Cloudsmith CLI - run: | - echo $(pip --version) - pip install --upgrade cloudsmith-cli - echo $(cloudsmith --version) - - - uses: goreleaser/goreleaser-action@v2 - name: Release, Upload & Publish - with: - version: latest - args: -f .publisher-ee.yml release --clean - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - REPO_NAME: ${{ github.repository }} - CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ebb4b5977b..197b3378c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: needs: [build_ui] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Download Build Artifact uses: actions/download-artifact@v2 with: @@ -40,22 +40,10 @@ jobs: path: api/ui/build fetch-depth: 0 - - uses: docker/login-action@v1 - name: Authenticate with Docker - with: - registry: docker.cloudsmith.io - username: ${{ secrets.CLOUDSMITH_USERNAME }} - password: ${{ secrets.CLOUDSMITH_API_KEY }} - - - uses: actions/setup-go@v2 - name: Setup go - with: - go-version: '1.21' - - uses: docker/setup-qemu-action@v3 name: Set up QEMU - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 name: Setup Python with: python-version: '3.9' diff --git a/.publisher-ee.yml b/.publisher-ee.yml deleted file mode 100644 index f8e722a27b..0000000000 --- a/.publisher-ee.yml +++ /dev/null @@ -1,100 +0,0 @@ -project_name: convoy - -before: - hooks: - - go mod tidy -builds: - - env: - - CGO_ENABLED=0 - main: ./ee/cmd - id: cobin - goos: - - linux - - darwin - - windows - goarch: - - amd64 - - arm64 - -# https://goreleaser.com/customization/archive/ -archives: - - name_template: "{{ .ProjectName}}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" - id: cobin-archive - builds: - - cobin - -dockers: - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-amd64" - use: buildx - goos: linux - goarch: amd64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=/~https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.source=/~https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-arm64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-arm64" - use: buildx - goos: linux - goarch: arm64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/arm64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=/~https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.source=/~https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-slim" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-slim" - goos: linux - goarch: amd64 - dockerfile: slim.Dockerfile - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=/~https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.source=/~https://github.com/{{ .Env.REPO_NAME }}-ee - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - -docker_manifests: - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:{{ .Tag }}-arm64" - - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}-ee:latest-arm64" - -checksum: - name_template: "{{ .ProjectName}}_checksums.txt" - -release: - # Will not auto-publish the release on GitHub - disable: true diff --git a/.publisher.yml b/.publisher.yml index 345f25d394..e496c2a668 100644 --- a/.publisher.yml +++ b/.publisher.yml @@ -87,76 +87,6 @@ brews: name: homebrew-tools url_template: https://dl.cloudsmith.io/public/convoy/convoy/raw/versions/{{.Version}}/{{ .ArtifactName }} -dockers: - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-amd64" - use: buildx - goos: linux - goarch: amd64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=/~https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.source=/~https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-arm64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-arm64" - use: buildx - goos: linux - goarch: arm64 - dockerfile: release.Dockerfile - extra_files: - - configs/local/start.sh - ids: - - cobin - build_flag_templates: - - --platform=linux/arm64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=/~https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.source=/~https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - - - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-slim" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-slim" - use: buildx - goos: linux - goarch: amd64 - dockerfile: slim.Dockerfile - ids: - - cobin - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title={{ .ProjectName }} - - --label=org.opencontainers.image.description=A fast & secure open source webhooks service - - --label=org.opencontainers.image.url=/~https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.source=/~https://github.com/{{ .Env.REPO_NAME }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.licenses=MPL-2.0 - -docker_manifests: - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:{{ .Tag }}-arm64" - - - name_template: "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest" - image_templates: - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-amd64" - - "docker.cloudsmith.io/convoy/convoy/{{ .Env.REPO_NAME }}:latest-arm64" - checksum: name_template: "{{ .ProjectName}}_checksums.txt" diff --git a/configs/convoy.templ.json b/configs/convoy.templ.json index 20ea2d4155..d9fc65086f 100644 --- a/configs/convoy.templ.json +++ b/configs/convoy.templ.json @@ -26,13 +26,6 @@ "password": "", "from": "support@frain.dev" }, - "search": { - "type": "typesense", - "typesense": { - "host": "http://typesense:8108", - "api_key": "convoy" - } - }, "server": { "http": { "ssl": false, diff --git a/configs/docker-compose.templ.yml b/configs/docker-compose.templ.yml index c7604b17d7..b3787fc020 100644 --- a/configs/docker-compose.templ.yml +++ b/configs/docker-compose.templ.yml @@ -2,7 +2,7 @@ version: "3" services: web: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: [ "/start.sh" ] hostname: web container_name: web @@ -12,12 +12,11 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster scheduler: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: ["./cmd", "scheduler", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -25,12 +24,11 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster worker: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: ["./cmd", "worker", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -38,12 +36,11 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster ingest: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:$VERSION + image: getconvoy/convoy:latest command: ["./cmd", "ingest", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -51,7 +48,6 @@ services: depends_on: - postgres - redis_server - - typesense networks: - backendCluster @@ -76,20 +72,6 @@ services: networks: - backendCluster - typesense: - image: typesense/typesense:0.22.2 - hostname: typesense - container_name: typesense - restart: always - environment: - TYPESENSE_DATA_DIR: /data/typesense - TYPESENSE_ENABLE_CORS: "true" - TYPESENSE_API_KEY: "convoy" - volumes: - - ./typesense-data:/data/typesense - networks: - - backendCluster - caddy: image: caddy restart: unless-stopped diff --git a/configs/local/docker-compose.yml b/configs/local/docker-compose.yml index cdd7eecba9..ffdd414f2a 100644 --- a/configs/local/docker-compose.yml +++ b/configs/local/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: web: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 + image: getconvoy/convoy:latest command: ["/start.sh"] volumes: - ./convoy.json:/convoy.json @@ -21,7 +21,7 @@ services: - pgbouncer worker: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 + image: getconvoy/convoy:latest command: ["./cmd", "worker", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -31,7 +31,7 @@ services: condition: service_healthy ingest: - image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 + image: getconvoy/convoy:latest command: ["./cmd", "ingest", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json diff --git a/datastore/filter.go b/datastore/filter.go index e50132a0ed..9bcb4c860b 100644 --- a/datastore/filter.go +++ b/datastore/filter.go @@ -46,10 +46,11 @@ type FilterBy struct { SearchParams SearchParams } -func (f *FilterBy) String() *string { +func (f *FilterBy) String() string { var s string filterByBuilder := new(strings.Builder) - filterByBuilder.WriteString(fmt.Sprintf("project_id:=%s", f.ProjectID)) // TODO(daniel, RT): how to work around this? + // TODO(daniel, raymond): how to work around this? + filterByBuilder.WriteString(fmt.Sprintf("project_id:=%s", f.ProjectID)) filterByBuilder.WriteString(fmt.Sprintf(" && created_at:[%d..%d]", f.SearchParams.CreatedAtStart, f.SearchParams.CreatedAtEnd)) if len(f.EndpointID) > 0 { @@ -62,9 +63,7 @@ func (f *FilterBy) String() *string { s = filterByBuilder.String() - // we only return a pointer address here - // because the typesense lib needs a string pointer - return &s + return s } type SearchFilter struct { diff --git a/datastore/filter_test.go b/datastore/filter_test.go index 867c798a73..d7c6f034d9 100644 --- a/datastore/filter_test.go +++ b/datastore/filter_test.go @@ -42,7 +42,7 @@ func Test_FilterBy(t *testing.T) { for _, tt := range args { t.Run(tt.name, func(t *testing.T) { s := tt.filter.String() - require.Equal(t, tt.expected, *s) + require.Equal(t, tt.expected, s) }) } } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 41a54390b7..2f186d8a68 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,7 +3,6 @@ version: "3" volumes: postgres_data: redis_data: - typesense_data: services: web: @@ -19,7 +18,6 @@ services: depends_on: - postgres - redis_server - - typesense scheduler: build: @@ -32,7 +30,6 @@ services: depends_on: - postgres - redis_server - - typesense worker: build: @@ -45,7 +42,6 @@ services: depends_on: - postgres - redis_server - - typesense ingest: build: @@ -58,7 +54,6 @@ services: depends_on: - postgres - redis_server - - typesense postgres: image: postgres:15.2-alpine @@ -77,16 +72,6 @@ services: volumes: - ./redis_data:/data - typesense: - image: typesense/typesense:0.22.2 - restart: always - environment: - TYPESENSE_DATA_DIR: /data/typesense - TYPESENSE_ENABLE_CORS: "true" - TYPESENSE_API_KEY: "convoy" - volumes: - - ./typesense_data:/data/typesense - prometheus: image: prom/prometheus:v2.24.0 volumes: diff --git a/release.Dockerfile b/release.Dockerfile index b24afca2c3..9d644c2113 100644 --- a/release.Dockerfile +++ b/release.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.16.2 +FROM alpine:3.20.2 COPY convoy /cmd COPY configs/local/start.sh /start.sh diff --git a/scripts/ui.sh b/scripts/ui.sh index b112c5986c..1271b258cb 100755 --- a/scripts/ui.sh +++ b/scripts/ui.sh @@ -12,7 +12,7 @@ buildUi() { cd ./web/ui/dashboard || exit 1 # Install dependencies - npm ci + npm i # Run production build if [[ "$build" == "ce" ]]; then diff --git a/worker/task/testdata/Config/basic-convoy.json b/worker/task/testdata/Config/basic-convoy.json index ab2f3fbcba..28e491efa4 100644 --- a/worker/task/testdata/Config/basic-convoy.json +++ b/worker/task/testdata/Config/basic-convoy.json @@ -47,12 +47,5 @@ "username": "apikey", "password": "", "from": "support@frain.dev" - }, - "search": { - "type": "typesense", - "typesense": { - "host": "http://localhost:8108", - "api_key": "convoy" - } } } From 6e14b5da6c555b21c15186a9dff9d1b46e605525 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 23 Aug 2024 19:08:11 +0200 Subject: [PATCH 07/16] Change License to Elastic License v2.0 (#2124) * feat: change to elv2 * feat: update README.md --- LICENSE | 389 ++++++++++-------------------------------------------- README.md | 2 +- 2 files changed, 73 insertions(+), 318 deletions(-) diff --git a/LICENSE b/LICENSE index c33dcc7c92..5a155b34d8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,354 +1,109 @@ -Mozilla Public License, version 2.0 +CONVOY - Copyright (C) 2021-2024 Frain Technologies INC. All rights reserved. -1. Definitions +BEFORE DOWNLOADING OR USING CONVOY (THE “SOFTWARE”), YOU SHOULD CAREFULLY +READ THE FOLLOWING LICENSE AGREEMENT THAT APPLIES TO YOUR USE OF THE SOFTWARE. +DOWNLOADING OR USING CONVOY ESTABLISHES A BINDING AGREEMENT BETWEEN FRAIN +TECHNOLOGIES INC. ("LICENSOR") AND YOU (INCLUDING YOUR COMPANY, IF +APPLICABLE). YOUR ACCEPTANCE OF THIS LICENSE AGREEMENT IS REQUIRED AS A +CONDITION TO PROCEEDING WITH YOUR DOWNLOAD OR USE OF THE SOFTWARE. -1.1. “Contributor” - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. +Elastic License 2.0 -1.2. “Contributor Version” +### Acceptance - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. +By using the software, you agree to all of the terms and conditions below. -1.3. “Contribution” - means Covered Software of a particular Contributor. +### Copyright License -1.4. “Covered Software” +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. -1.5. “Incompatible With Secondary Licenses” - means +### Limitations - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. -1.6. “Executable Form” +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. - means any form of the work other than Source Code Form. -1.7. “Larger Work” +### Patents - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. -1.8. “License” - means this document. +### Notices -1.9. “Licensable” +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. -1.10. “Modifications” - means any of the following: +### No Other Rights - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or +These terms do not imply any licenses other than those expressly granted in +these terms. - b. any new file in Source Code Form that contains any Covered Software. -1.11. “Patent Claims” of a Contributor +### Termination - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. -1.12. “Secondary License” - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. +### No Liability -1.13. “Source Code Form” +As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim. - means the form of the work preferred for making modifications. -1.14. “You” (or “Your”) +### Definitions - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. +**you** refers to the individual or entity agreeing to these terms. -2. License Grants and Conditions +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. 'control' means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. -2.1. Grants +**your licenses** are all the licenses granted to you for the software under +these terms. - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and - - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. - -2.2. Effective Date - - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. - -2.3. Limitations on Grant Scope - - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: - - a. for any code that a Contributor has removed from Covered Software; or - - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. - - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). - -2.4. Subsequent Licenses - - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). - -2.5. Representation - - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. - -2.6. Fair Use - - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. - - -3. Responsibilities - -3.1. Distribution of Source Form - - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - - If You distribute Covered Software in Executable Form then: - - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and - - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). - -3.4. Notices - - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. - -6. Disclaimer of Warranty - - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation - - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. - - -10. Versions of the License - -10.1. New Versions - - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. - -10.2. Effect of New Versions - - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. - -10.3. Modified Versions - - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. +**use** means anything you do with the software requiring one of your licenses. +**trademark** means trademarks, service marks, and similar rights. \ No newline at end of file diff --git a/README.md b/README.md index bc985e97bd..a437e8b852 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,4 @@ Convoy provides several key features: Thank you for your interest in contributing! Please refer to [CONTRIBUTING.md](/~https://github.com/frain-dev/convoy/blob/main/CONTRIBUTING.md) for guidance. For contributions to the Convoy dashboard, please refer to the [web/ui](/~https://github.com/frain-dev/convoy/tree/main/web/ui) directory. ## License -[Mozilla Public License v2.0](/~https://github.com/frain-dev/convoy/blob/main/LICENSE) +[Elastic License v2.0](/~https://github.com/frain-dev/convoy/blob/main/LICENSE) From eb50a3c0fa83c1fc0292905f2ee34fedeb86fda8 Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Mon, 26 Aug 2024 18:35:36 +0200 Subject: [PATCH 08/16] chore: update image (#2126) --- configs/local/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/local/docker-compose.yml b/configs/local/docker-compose.yml index ffdd414f2a..cdd7eecba9 100644 --- a/configs/local/docker-compose.yml +++ b/configs/local/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: web: - image: getconvoy/convoy:latest + image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 command: ["/start.sh"] volumes: - ./convoy.json:/convoy.json @@ -21,7 +21,7 @@ services: - pgbouncer worker: - image: getconvoy/convoy:latest + image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 command: ["./cmd", "worker", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json @@ -31,7 +31,7 @@ services: condition: service_healthy ingest: - image: getconvoy/convoy:latest + image: docker.cloudsmith.io/convoy/convoy/frain-dev/convoy:v24.6.4 command: ["./cmd", "ingest", "--config", "convoy.json"] volumes: - ./convoy.json:/convoy.json From 887265368a3cbf7c9b196e5feed1a9493e7bd19e Mon Sep 17 00:00:00 2001 From: Raymond Tukpe Date: Fri, 30 Aug 2024 13:12:05 +0200 Subject: [PATCH 09/16] patch: add owner id to event delivery response (#2129) --- database/postgres/event_delivery.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/database/postgres/event_delivery.go b/database/postgres/event_delivery.go index e138385374..2d82aa6b88 100644 --- a/database/postgres/event_delivery.go +++ b/database/postgres/event_delivery.go @@ -61,6 +61,8 @@ const ( COALESCE(ep.project_id, '') AS "endpoint_metadata.project_id", COALESCE(ep.support_email, '') AS "endpoint_metadata.support_email", COALESCE(ep.url, '') AS "endpoint_metadata.url", + COALESCE(ep.owner_id, '') AS "endpoint_metadata.owner_id", + ev.id AS "event_metadata.id", ev.event_type AS "event_metadata.event_type", COALESCE(ed.latency_seconds, 0) AS latency_seconds, @@ -738,6 +740,7 @@ func (e *eventDeliveryRepo) LoadEventDeliveriesPaged(ctx context.Context, projec Url: ev.Endpoint.URL.ValueOrZero(), Name: ev.Endpoint.Name.ValueOrZero(), SupportEmail: ev.Endpoint.SupportEmail.ValueOrZero(), + OwnerID: ev.Endpoint.OwnerID.ValueOrZero(), }, Source: &datastore.Source{ UID: ev.Source.UID.ValueOrZero(), @@ -945,6 +948,7 @@ type EndpointMetadata struct { URL null.String `db:"url"` ProjectID null.String `db:"project_id"` SupportEmail null.String `db:"support_email"` + OwnerID null.String `db:"owner_id"` } type EventMetadata struct { From 6380f177e8838d164eb4a07f913dd3d14ab599b1 Mon Sep 17 00:00:00 2001 From: Daniel Oluojomu Date: Fri, 30 Aug 2024 15:11:03 +0100 Subject: [PATCH 10/16] Add license feature gating (#2114) * dump * feat: add keygen licenser implementation * feat: gate more features * feat: fix & add tests for licensing * feat: fix what test * fix: remove unused import * fix: add skipHook * fix: fix PostRun panic * feat: fix task tests * fix: - pass licenser fields to services - fix organisation_member_test.go & organisation_test.go * fix: add license key env for testcon tests * fix: fix license error check * fix: add test license key env to docker-compose-test.yml * fix: add CONVOY_LICENSE_KEY to integration_test_helper.go * fix: print TEST_LICENSE_KEY * fix: remove print * fix: print * fix: print license key * fix: pass license key in docker-compose-test.yml * fix: - remove print - use withEnv in integration_test.go * fix: add keygen_test.go * fix: remove plan_type validation * fix: make TestKeygenLicenserBoolMethods more insulated * fix: add TestNewDispatcher * deprecate: deprecate RedirectToProjects for old convoy routes * fix: - pass licenser to UpdateEndpointService - check for advanced endpoint mgmt before calling SendRequest * fix: add CommunityPlan & communityLicenser * fix: extract feature list from license entitlements * fix: add GET /ui/license_features api * fix: go generate * fix: move TEST_LICENSE_KEY to Run integration tests (with test containers) * fix: extract license limits from metadata * fix: move /ui/license_features to BuildControlPlaneRoutes * fix: fix GetLicenseFeatures response msg * fix: use json.RawMessage as return type for FeatureListJSON method * fix: - change /ui/license_features to /ui/license/features - add integration build tag to noop.go - check compose error * fix: fix lint issues * fix: edit some feature names * fix: change keygen creds * fix: check compose error for container exited with code 0 * feat: gate project creation * feat: add omitempty tag to Properties.Limit * feat: change CreateOrgMember gating to Register user gating * fix: fix TestCountProjects * feat: gate portal links * fix: - pass projectRepo to licenser from hooks.go - fix ProjectLimit mapstructure tag * fix: - switch to counting users from org members - gate boostrap command * fix: fix Test_communityLicenser * fix: - remove TestCountOrganisationMembers - add TestCountUsers * fix: fix TestCountUsers * add license gating (#2123) * add licensing * update license tags * fix: use userRepo.CountUsers in ensureDefaultUser * fix: remove CreateUser from commnunity licenser * fix: fix Test_communityLicenser * add portal links gating * fix: switch feature list json to `allowed: true/false` because of features with dynamic limits * fix: go generate * fix: fix lint issues * fix: fix /license/features in guestRoutes * check license limits * remove console.log * fetch licenses on org creation * update user locense gating * update license gating on projects, teams, transform and filters * fix: check license expiry * Disable projects when user downgrades (#2128) * fix: - add RemoveEnabledProject - check enabledProjects under limit before adding new project * fix: add test case for RemoveEnabledProject * fix: gate portal link api in data plane routes * fix: change ErrProjectDisabled err msg --------- Co-authored-by: Pelumi Muyiwa-Oni --- .github/workflows/go.yml | 1 + api/api.go | 333 +++++---- api/handlers/endpoint.go | 2 + api/handlers/license.go | 21 + api/handlers/middleware.go | 30 + api/handlers/organisation.go | 1 + api/handlers/organisation_invite.go | 2 + api/handlers/project.go | 5 +- api/handlers/shim.go | 67 -- api/handlers/subscription.go | 12 + api/handlers/user.go | 6 +- api/ingest.go | 14 +- api/ingest_integration_test.go | 1 + api/server_suite_test.go | 21 +- api/types/types.go | 16 +- cmd/agent/agent.go | 18 +- cmd/bootstrap/bootstrap.go | 9 + cmd/hooks/hooks.go | 60 +- cmd/ingest/ingest.go | 3 +- cmd/main.go | 3 + cmd/server/server.go | 13 +- cmd/worker/worker.go | 13 +- config/config.go | 1 + database/postgres/organisation.go | 19 +- database/postgres/organisation_member.go | 1 + database/postgres/organisation_test.go | 27 + database/postgres/project.go | 18 +- database/postgres/project_test.go | 26 + database/postgres/users.go | 120 +--- database/postgres/users_test.go | 95 +-- datastore/repository.go | 4 +- docs/docs.go | 5 +- docs/swagger.json | 3 + docs/swagger.yaml | 2 + docs/v3/openapi3.json | 3 + docs/v3/openapi3.yaml | 2 + ee/cmd/server/server.go | 16 +- generate.go | 1 + go.mod | 74 +- go.sum | 154 ++-- internal/pkg/cli/cli.go | 15 +- internal/pkg/license/keygen/community.go | 55 ++ internal/pkg/license/keygen/community_test.go | 89 +++ internal/pkg/license/keygen/feature.go | 44 ++ internal/pkg/license/keygen/keygen.go | 456 ++++++++++++ internal/pkg/license/keygen/keygen_test.go | 658 ++++++++++++++++++ internal/pkg/license/license.go | 45 ++ internal/pkg/license/noop/noop.go | 89 +++ internal/pkg/metrics/data_plane.go | 23 +- internal/pkg/middleware/middleware.go | 7 +- internal/pkg/pubsub/amqp/client.go | 13 +- internal/pkg/pubsub/google/client.go | 9 +- internal/pkg/pubsub/ingest.go | 10 +- internal/pkg/pubsub/kafka/client.go | 11 +- internal/pkg/pubsub/pubsub.go | 22 +- internal/pkg/pubsub/sqs/client.go | 16 +- mocks/license.go | 321 +++++++++ mocks/repository.go | 61 +- net/dispatcher.go | 18 +- net/dispatcher_test.go | 72 ++ services/create_endpoint.go | 10 + services/create_endpoint_test.go | 68 +- services/create_organisation.go | 17 +- services/create_organisation_test.go | 25 + services/create_subscription.go | 17 +- services/create_subscription_test.go | 98 +++ services/invite_user.go | 16 +- services/invite_user_test.go | 22 + services/process_invite.go | 14 + services/process_invite_test.go | 54 ++ services/project_service.go | 24 +- services/project_service_test.go | 56 +- services/register_user.go | 19 +- services/register_user_test.go | 42 ++ services/update_endpoint.go | 13 +- services/update_endpoint_test.go | 47 ++ services/update_subscription.go | 14 +- services/update_subscription_test.go | 1 + testcon/docker_e2e_integration_test.go | 19 +- testcon/docker_e2e_integration_test_helper.go | 25 +- testcon/testdata/docker-compose-test.yml | 4 + .../src/app/components/tag/tag.component.ts | 3 +- .../create-endpoint.component.html | 18 +- .../create-endpoint.component.ts | 8 +- .../create-project-component.component.ts | 30 +- .../create-source.component.html | 26 +- .../create-source/create-source.component.ts | 3 +- .../create-source/create-source.module.ts | 4 +- .../create-subscription.component.html | 26 +- .../create-subscription.component.ts | 3 +- .../create-subscription.module.ts | 4 +- .../portal-links/portal-links.component.html | 257 +++---- .../portal-links/portal-links.component.ts | 5 +- .../pages/project/project.component.html | 31 +- .../pages/project/project.component.ts | 3 +- .../subscriptions.component.html | 6 +- .../subscriptions/subscriptions.component.ts | 3 +- .../pages/projects/projects.component.html | 20 +- .../pages/projects/projects.component.ts | 3 +- .../organisation-settings.component.html | 2 +- .../organisation-settings.component.ts | 3 +- .../pages/settings/settings.component.html | 16 +- .../pages/settings/settings.component.ts | 13 +- .../src/app/private/private.component.html | 8 +- .../src/app/private/private.component.ts | 12 +- .../src/app/public/login/login.component.html | 2 +- .../src/app/public/login/login.component.ts | 4 +- .../src/app/public/signup/signup.component.ts | 9 +- .../licenses/licenses.service.spec.ts | 16 + .../app/services/licenses/licenses.service.ts | 51 ++ .../src/assets/img/svg/page-locked.svg | 9 + web/ui/dashboard/src/index.html | 6 + .../task/process_broadcast_event_creation.go | 10 +- .../process_broadcast_event_creation_test.go | 2 +- worker/task/process_dynamic_event_creation.go | 8 +- .../process_dynamic_event_creation_test.go | 2 +- worker/task/process_event_creation.go | 26 +- worker/task/process_event_creation_test.go | 159 ++++- worker/task/process_event_delivery.go | 31 +- worker/task/process_event_delivery_test.go | 144 +++- worker/task/process_meta_event.go | 14 +- worker/task/process_meta_event_test.go | 13 +- .../task/process_retry_event_delivery_test.go | 166 ++++- 123 files changed, 4067 insertions(+), 973 deletions(-) create mode 100644 api/handlers/license.go create mode 100644 api/handlers/middleware.go delete mode 100644 api/handlers/shim.go create mode 100644 internal/pkg/license/keygen/community.go create mode 100644 internal/pkg/license/keygen/community_test.go create mode 100644 internal/pkg/license/keygen/feature.go create mode 100644 internal/pkg/license/keygen/keygen.go create mode 100644 internal/pkg/license/keygen/keygen_test.go create mode 100644 internal/pkg/license/license.go create mode 100644 internal/pkg/license/noop/noop.go create mode 100644 mocks/license.go create mode 100644 web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts create mode 100644 web/ui/dashboard/src/app/services/licenses/licenses.service.ts create mode 100644 web/ui/dashboard/src/assets/img/svg/page-locked.svg diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d9d834f1da..6b9179619c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -94,6 +94,7 @@ jobs: - name: Run integration tests (with test containers) run: make docker_e2e_tests env: + TEST_LICENSE_KEY: ${{ secrets.CONVOY_TEST_LICENSE_KEY }} TEST_DB_SCHEME: postgres TEST_DB_HOST: localhost TEST_DB_USERNAME: postgres diff --git a/api/api.go b/api/api.go index 358db79250..e868e60d84 100644 --- a/api/api.go +++ b/api/api.go @@ -2,6 +2,11 @@ package api import ( "embed" + "io/fs" + "net/http" + "path" + "strings" + authz "github.com/Subomi/go-authz" "github.com/frain-dev/convoy/api/handlers" "github.com/frain-dev/convoy/api/policies" @@ -13,13 +18,8 @@ import ( redisqueue "github.com/frain-dev/convoy/queue/redis" "github.com/go-chi/chi/v5" chiMiddleware "github.com/go-chi/chi/v5/middleware" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/subomi/requestmigrations" - "io/fs" - "net/http" - "path" - "strings" ) //go:embed ui/build @@ -119,7 +119,6 @@ func (a *ApplicationHandler) buildRouter() *chi.Mux { } func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { - router := a.buildRouter() handler := &handlers.Handler{A: a.A, RM: a.rm} @@ -145,46 +144,54 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { projectRouter.Route("/{projectID}", func(projectSubRouter chi.Router) { projectSubRouter.Get("/", handler.GetProject) - projectSubRouter.Put("/", handler.UpdateProject) + projectSubRouter.With(handler.RequireEnabledProject()).Put("/", handler.UpdateProject) projectSubRouter.Delete("/", handler.DeleteProject) projectSubRouter.Route("/endpoints", func(endpointSubRouter chi.Router) { - endpointSubRouter.Post("/", handler.CreateEndpoint) + endpointSubRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateEndpoint) endpointSubRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) endpointSubRouter.Route("/{endpointID}", func(e chi.Router) { e.Get("/", handler.GetEndpoint) - e.Put("/", handler.UpdateEndpoint) - e.Delete("/", handler.DeleteEndpoint) - e.Put("/expire_secret", handler.ExpireSecret) - e.Put("/pause", handler.PauseEndpoint) + + e.With(handler.RequireEnabledProject()).Use(handler.RequireEnabledProject()) + + e.With(handler.RequireEnabledProject()).Put("/", handler.UpdateEndpoint) + e.With(handler.RequireEnabledProject()).Delete("/", handler.DeleteEndpoint) + e.With(handler.RequireEnabledProject()).Put("/expire_secret", handler.ExpireSecret) + e.With(handler.RequireEnabledProject()).Put("/pause", handler.PauseEndpoint) }) }) // TODO(subomi): left this here temporarily till the data plane is stable. projectSubRouter.Route("/events", func(eventRouter chi.Router) { - // TODO(all): should the InstrumentPath change? - eventRouter.With(middleware.InstrumentPath("/events")).Post("/", handler.CreateEndpointEvent) - eventRouter.Post("/fanout", handler.CreateEndpointFanoutEvent) - eventRouter.Post("/broadcast", handler.CreateBroadcastEvent) - eventRouter.Post("/dynamic", handler.CreateDynamicEvent) - eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Route("/", func(writeEventRouter chi.Router) { + eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) - eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { - eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + // TODO(all): should the InstrumentPath change? + eventRouter.With(handler.RequireEnabledProject(), middleware.InstrumentPath("/events")).Post("/", handler.CreateEndpointEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/fanout", handler.CreateEndpointFanoutEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/broadcast", handler.CreateBroadcastEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/dynamic", handler.CreateDynamicEvent) + eventRouter.With(handler.RequireEnabledProject()).With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.With(handler.RequireEnabledProject()).Post("/batchreplay", handler.BatchReplayEvents) + + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.With(handler.RequireEnabledProject()).Put("/replay", handler.ReplayEndpointEvent) + eventSubRouter.Get("/", handler.GetEndpointEvent) + }) }) }) projectSubRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/batchretry", handler.BatchRetryEventDelivery) eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliverySubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendEventDelivery) eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { deliveryRouter.Get("/", handler.GetDeliveryAttempts) @@ -194,45 +201,45 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) projectSubRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { - subscriptionRouter.Post("/", handler.CreateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSubscription) subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) subscriptionRouter.Post("/test_function", handler.TestSubscriptionFunction) subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) - subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Delete("/{subscriptionID}", handler.DeleteSubscription) subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) - subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Put("/{subscriptionID}", handler.UpdateSubscription) subscriptionRouter.Put("/{subscriptionID}/toggle_status", handler.ToggleSubscriptionStatus) }) projectSubRouter.Route("/sources", func(sourceRouter chi.Router) { - sourceRouter.Post("/", handler.CreateSource) + sourceRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSource) sourceRouter.Get("/{sourceID}", handler.GetSource) sourceRouter.With(middleware.Pagination).Get("/", handler.LoadSourcesPaged) sourceRouter.Post("/test_function", handler.TestSourceFunction) - sourceRouter.Put("/{sourceID}", handler.UpdateSource) - sourceRouter.Delete("/{sourceID}", handler.DeleteSource) + sourceRouter.With(handler.RequireEnabledProject()).Put("/{sourceID}", handler.UpdateSource) + sourceRouter.With(handler.RequireEnabledProject()).Delete("/{sourceID}", handler.DeleteSource) }) - projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { - portalLinkRouter.Post("/", handler.CreatePortalLink) - portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) - portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) - portalLinkRouter.Put("/{portalLinkID}", handler.UpdatePortalLink) - portalLinkRouter.Put("/{portalLinkID}/revoke", handler.RevokePortalLink) - }) + if handler.A.Licenser.PortalLinks() { + projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { + portalLinkRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreatePortalLink) + portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) + portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) + portalLinkRouter.With(handler.RequireEnabledProject()).Put("/{portalLinkID}", handler.UpdatePortalLink) + portalLinkRouter.With(handler.RequireEnabledProject()).Put("/{portalLinkID}/revoke", handler.RevokePortalLink) + }) + } projectSubRouter.Route("/meta-events", func(metaEventRouter chi.Router) { metaEventRouter.With(middleware.Pagination).Get("/", handler.GetMetaEventsPaged) metaEventRouter.Route("/{metaEventID}", func(metaEventSubRouter chi.Router) { metaEventSubRouter.Get("/", handler.GetMetaEvent) - metaEventSubRouter.Put("/resend", handler.ResendMetaEvent) + metaEventSubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendMetaEvent) }) }) }) }) - - r.HandleFunc("/*", handler.RedirectToProjects) }) }) @@ -241,6 +248,8 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { uiRouter.Use(middleware.JsonResponse) uiRouter.Use(chiMiddleware.Maybe(middleware.RequireAuth(), shouldAuthRoute)) + uiRouter.Get("/license/features", handler.GetLicenseFeatures) + uiRouter.Post("/users/forgot-password", handler.ForgotPassword) uiRouter.Post("/users/reset-password", handler.ResetPassword) uiRouter.Post("/users/verify_email", handler.VerifyEmail) @@ -300,50 +309,58 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { projectRouter.Route("/{projectID}", func(projectSubRouter chi.Router) { projectSubRouter.Get("/", handler.GetProject) - projectSubRouter.Put("/", handler.UpdateProject) - projectSubRouter.Delete("/", handler.DeleteProject) + projectSubRouter.With(handler.RequireEnabledProject()).Put("/", handler.UpdateProject) + projectSubRouter.With(handler.RequireEnabledProject()).Delete("/", handler.DeleteProject) projectSubRouter.Get("/stats", handler.GetProjectStatistics) projectSubRouter.Route("/security/keys", func(projectKeySubRouter chi.Router) { - projectKeySubRouter.Put("/regenerate", handler.RegenerateProjectAPIKey) + projectKeySubRouter.With(handler.RequireEnabledProject()).Put("/regenerate", handler.RegenerateProjectAPIKey) }) projectSubRouter.Route("/endpoints", func(endpointSubRouter chi.Router) { - endpointSubRouter.Post("/", handler.CreateEndpoint) + endpointSubRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateEndpoint) endpointSubRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) endpointSubRouter.Route("/{endpointID}", func(e chi.Router) { e.Get("/", handler.GetEndpoint) - e.Put("/", handler.UpdateEndpoint) - e.Delete("/", handler.DeleteEndpoint) - e.Put("/expire_secret", handler.ExpireSecret) - e.Put("/pause", handler.PauseEndpoint) + + e.With(handler.RequireEnabledProject()).Use(handler.RequireEnabledProject()) + + e.With(handler.RequireEnabledProject()).Put("/", handler.UpdateEndpoint) + e.With(handler.RequireEnabledProject()).Delete("/", handler.DeleteEndpoint) + e.With(handler.RequireEnabledProject()).Put("/expire_secret", handler.ExpireSecret) + e.With(handler.RequireEnabledProject()).Put("/pause", handler.PauseEndpoint) }) }) // TODO(subomi): left this here temporarily till the data plane is stable. projectSubRouter.Route("/events", func(eventRouter chi.Router) { - eventRouter.Post("/", handler.CreateEndpointEvent) - eventRouter.Post("/fanout", handler.CreateEndpointFanoutEvent) eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) + // TODO(all): should the InstrumentPath change? + eventRouter.With(handler.RequireEnabledProject(), middleware.InstrumentPath("/events")).Post("/", handler.CreateEndpointEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/fanout", handler.CreateEndpointFanoutEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/broadcast", handler.CreateBroadcastEvent) + eventRouter.With(handler.RequireEnabledProject()).Post("/dynamic", handler.CreateDynamicEvent) + eventRouter.With(handler.RequireEnabledProject()).With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.With(handler.RequireEnabledProject()).Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.With(handler.RequireEnabledProject()).Put("/replay", handler.ReplayEndpointEvent) eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) }) }) projectSubRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.With(handler.RequireEnabledProject()).Post("/batchretry", handler.BatchRetryEventDelivery) eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliverySubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendEventDelivery) eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { deliveryRouter.Get("/", handler.GetDeliveryAttempts) @@ -353,22 +370,22 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) projectSubRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { - subscriptionRouter.Post("/", handler.CreateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSubscription) subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) subscriptionRouter.Post("/test_function", handler.TestSubscriptionFunction) subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) - subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Delete("/{subscriptionID}", handler.DeleteSubscription) subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) - subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + subscriptionRouter.With(handler.RequireEnabledProject()).Put("/{subscriptionID}", handler.UpdateSubscription) }) projectSubRouter.Route("/sources", func(sourceRouter chi.Router) { - sourceRouter.Post("/", handler.CreateSource) + sourceRouter.With(handler.RequireEnabledProject()).Post("/", handler.CreateSource) sourceRouter.Get("/{sourceID}", handler.GetSource) sourceRouter.With(middleware.Pagination).Get("/", handler.LoadSourcesPaged) sourceRouter.Post("/test_function", handler.TestSourceFunction) - sourceRouter.Put("/{sourceID}", handler.UpdateSource) - sourceRouter.Delete("/{sourceID}", handler.DeleteSource) + sourceRouter.With(handler.RequireEnabledProject()).Put("/{sourceID}", handler.UpdateSource) + sourceRouter.With(handler.RequireEnabledProject()).Delete("/{sourceID}", handler.DeleteSource) }) projectSubRouter.Route("/meta-events", func(metaEventRouter chi.Router) { @@ -376,17 +393,19 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { metaEventRouter.Route("/{metaEventID}", func(metaEventSubRouter chi.Router) { metaEventSubRouter.Get("/", handler.GetMetaEvent) - metaEventSubRouter.Put("/resend", handler.ResendMetaEvent) + metaEventSubRouter.With(handler.RequireEnabledProject()).Put("/resend", handler.ResendMetaEvent) }) }) - projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { - portalLinkRouter.Post("/", handler.CreatePortalLink) - portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) - portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) - portalLinkRouter.Put("/{portalLinkID}", handler.UpdatePortalLink) - portalLinkRouter.Put("/{portalLinkID}/revoke", handler.RevokePortalLink) - }) + if handler.A.Licenser.PortalLinks() { + projectSubRouter.Route("/portal-links", func(portalLinkRouter chi.Router) { + portalLinkRouter.Post("/", handler.CreatePortalLink) + portalLinkRouter.Get("/{portalLinkID}", handler.GetPortalLink) + portalLinkRouter.With(middleware.Pagination).Get("/", handler.LoadPortalLinksPaged) + portalLinkRouter.Put("/{portalLinkID}", handler.UpdatePortalLink) + portalLinkRouter.Put("/{portalLinkID}/revoke", handler.RevokePortalLink) + }) + } projectSubRouter.Route("/dashboard", func(dashboardRouter chi.Router) { dashboardRouter.Get("/summary", handler.GetDashboardSummary) @@ -403,78 +422,81 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { }) // Portal Link API. - router.Route("/portal-api", func(portalLinkRouter chi.Router) { - portalLinkRouter.Use(middleware.JsonResponse) - portalLinkRouter.Use(middleware.SetupCORS) - portalLinkRouter.Use(middleware.RequireAuth()) - - portalLinkRouter.Get("/portal_link", handler.GetPortalLink) - - portalLinkRouter.Route("/endpoints", func(endpointRouter chi.Router) { - endpointRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) - endpointRouter.Get("/{endpointID}", handler.GetEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Post("/", handler.CreateEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}", handler.UpdateEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Delete("/{endpointID}", handler.DeleteEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/pause", handler.PauseEndpoint) - endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/expire_secret", handler.ExpireSecret) - }) + if handler.A.Licenser.PortalLinks() { + router.Route("/portal-api", func(portalLinkRouter chi.Router) { + portalLinkRouter.Use(middleware.JsonResponse) + portalLinkRouter.Use(middleware.SetupCORS) + portalLinkRouter.Use(middleware.RequireAuth()) + + portalLinkRouter.Get("/portal_link", handler.GetPortalLink) + + portalLinkRouter.Route("/endpoints", func(endpointRouter chi.Router) { + endpointRouter.With(middleware.Pagination).Get("/", handler.GetEndpoints) + endpointRouter.Get("/{endpointID}", handler.GetEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Post("/", handler.CreateEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}", handler.UpdateEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Delete("/{endpointID}", handler.DeleteEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/pause", handler.PauseEndpoint) + endpointRouter.With(handler.CanManageEndpoint()).Put("/{endpointID}/expire_secret", handler.ExpireSecret) + }) - // TODO(subomi): left this here temporarily till the data plane is stable. - portalLinkRouter.Route("/events", func(eventRouter chi.Router) { - eventRouter.Post("/", handler.CreateEndpointEvent) - eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) - eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) + // TODO(subomi): left this here temporarily till the data plane is stable. + portalLinkRouter.Route("/events", func(eventRouter chi.Router) { + eventRouter.Post("/", handler.CreateEndpointEvent) + eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) - eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { - eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.Get("/", handler.GetEndpointEvent) + eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + }) }) - }) - portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { - eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) - eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) + portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { + eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) + eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) - eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { - eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { + eventDeliverySubRouter.Get("/", handler.GetEventDelivery) + eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) - eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { - deliveryRouter.Get("/", handler.GetDeliveryAttempts) - deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { + deliveryRouter.Get("/", handler.GetDeliveryAttempts) + deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + }) }) }) - }) - portalLinkRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { - subscriptionRouter.Post("/", handler.CreateSubscription) - subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) - subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) - subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) - subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) - subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + portalLinkRouter.Route("/subscriptions", func(subscriptionRouter chi.Router) { + subscriptionRouter.Post("/", handler.CreateSubscription) + subscriptionRouter.Post("/test_filter", handler.TestSubscriptionFilter) + subscriptionRouter.With(middleware.Pagination).Get("/", handler.GetSubscriptions) + subscriptionRouter.Delete("/{subscriptionID}", handler.DeleteSubscription) + subscriptionRouter.Get("/{subscriptionID}", handler.GetSubscription) + subscriptionRouter.Put("/{subscriptionID}", handler.UpdateSubscription) + }) }) + } - }) - - router.Route("/metrics", func(metricsRouter chi.Router) { - metricsRouter.Use(middleware.RequireAuth()) - metricsRouter.Get("/", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()}).ServeHTTP) - }) + if a.A.Licenser.AsynqMonitoring() { + router.Route("/queue", func(asynqRouter chi.Router) { + asynqRouter.Use(middleware.RequireAuth()) + asynqRouter.Handle("/monitoring/*", a.A.Queue.(*redisqueue.RedisQueue).Monitor()) + }) + } - router.Route("/queue", func(metricsRouter chi.Router) { - metricsRouter.Use(middleware.RequireAuth()) - metricsRouter.Handle("/monitoring/*", a.A.Queue.(*redisqueue.RedisQueue).Monitor()) - }) + if a.A.Licenser.CanExportPrometheusMetrics() { + router.Route("/metrics", func(metricsRouter chi.Router) { + metricsRouter.Use(middleware.RequireAuth()) + metricsRouter.Get("/", promhttp.HandlerFor(metrics.Reg(), promhttp.HandlerOpts{Registry: metrics.Reg()}).ServeHTTP) + }) + } router.HandleFunc("/*", reactRootHandler) - metrics.RegisterQueueMetrics(a.A.Queue, a.A.DB) - prometheus.MustRegister(metrics.RequestDuration()) a.Router = router return router @@ -594,40 +616,42 @@ func (a *ApplicationHandler) BuildDataPlaneRoutes() *chi.Mux { }) // Portal Link API. - router.Route("/portal-api", func(portalLinkRouter chi.Router) { - portalLinkRouter.Use(middleware.JsonResponse) - portalLinkRouter.Use(middleware.SetupCORS) - portalLinkRouter.Use(middleware.RequireAuth()) - - portalLinkRouter.Route("/events", func(eventRouter chi.Router) { - eventRouter.Post("/", handler.CreateEndpointEvent) - eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) - eventRouter.Post("/batchreplay", handler.BatchReplayEvents) - eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) - - eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { - eventSubRouter.Get("/", handler.GetEndpointEvent) - eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + if handler.A.Licenser.PortalLinks() { + router.Route("/portal-api", func(portalLinkRouter chi.Router) { + portalLinkRouter.Use(middleware.JsonResponse) + portalLinkRouter.Use(middleware.SetupCORS) + portalLinkRouter.Use(middleware.RequireAuth()) + + portalLinkRouter.Route("/events", func(eventRouter chi.Router) { + eventRouter.Post("/", handler.CreateEndpointEvent) + eventRouter.With(middleware.Pagination).Get("/", handler.GetEventsPaged) + eventRouter.Post("/batchreplay", handler.BatchReplayEvents) + eventRouter.Get("/countbatchreplayevents", handler.CountAffectedEvents) + + eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { + eventSubRouter.Get("/", handler.GetEndpointEvent) + eventSubRouter.Put("/replay", handler.ReplayEndpointEvent) + }) }) - }) - portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { - eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) - eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) - eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) - eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) + portalLinkRouter.Route("/eventdeliveries", func(eventDeliveryRouter chi.Router) { + eventDeliveryRouter.With(middleware.Pagination).Get("/", handler.GetEventDeliveriesPaged) + eventDeliveryRouter.Post("/forceresend", handler.ForceResendEventDeliveries) + eventDeliveryRouter.Post("/batchretry", handler.BatchRetryEventDelivery) + eventDeliveryRouter.Get("/countbatchretryevents", handler.CountAffectedEventDeliveries) - eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { - eventDeliverySubRouter.Get("/", handler.GetEventDelivery) - eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) + eventDeliveryRouter.Route("/{eventDeliveryID}", func(eventDeliverySubRouter chi.Router) { + eventDeliverySubRouter.Get("/", handler.GetEventDelivery) + eventDeliverySubRouter.Put("/resend", handler.ResendEventDelivery) - eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { - deliveryRouter.Get("/", handler.GetDeliveryAttempts) - deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + eventDeliverySubRouter.Route("/deliveryattempts", func(deliveryRouter chi.Router) { + deliveryRouter.Get("/", handler.GetDeliveryAttempts) + deliveryRouter.Get("/{deliveryAttemptID}", handler.GetDeliveryAttempt) + }) }) }) }) - }) + } a.Router = router @@ -677,6 +701,7 @@ var guestRoutes = []string{ "/users/verify_email", "/organisations/process_invite", "/ui/configuration/is_signup_enabled", + "/ui/license/features", } func shouldAuthRoute(r *http.Request) bool { diff --git a/api/handlers/endpoint.go b/api/handlers/endpoint.go index c57895c58f..28103ff580 100644 --- a/api/handlers/endpoint.go +++ b/api/handlers/endpoint.go @@ -75,6 +75,7 @@ func (h *Handler) CreateEndpoint(w http.ResponseWriter, r *http.Request) { EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), ProjectRepo: postgres.NewProjectRepo(h.A.DB, h.A.Cache), PortalLinkRepo: postgres.NewPortalLinkRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, E: e, ProjectID: project.UID, } @@ -281,6 +282,7 @@ func (h *Handler) UpdateEndpoint(w http.ResponseWriter, r *http.Request) { Cache: h.A.Cache, EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), ProjectRepo: postgres.NewProjectRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, E: e, Endpoint: endpoint, Project: project, diff --git a/api/handlers/license.go b/api/handlers/license.go new file mode 100644 index 0000000000..968764b2e0 --- /dev/null +++ b/api/handlers/license.go @@ -0,0 +1,21 @@ +package handlers + +import ( + "net/http" + + "github.com/frain-dev/convoy/pkg/log" + + "github.com/frain-dev/convoy/util" + "github.com/go-chi/render" +) + +func (h *Handler) GetLicenseFeatures(w http.ResponseWriter, r *http.Request) { + v, err := h.A.Licenser.FeatureListJSON(r.Context()) + if err != nil { + log.FromContext(r.Context()).WithError(err).Error("failed to get license features") + _ = render.Render(w, r, util.NewErrorResponse("failed to get license features", http.StatusBadRequest)) + return + } + + _ = render.Render(w, r, util.NewServerResponse("Retrieved license features successfully", v, http.StatusOK)) +} diff --git a/api/handlers/middleware.go b/api/handlers/middleware.go new file mode 100644 index 0000000000..a314ed7ce9 --- /dev/null +++ b/api/handlers/middleware.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "errors" + "net/http" + + "github.com/frain-dev/convoy/util" + "github.com/go-chi/render" +) + +var ErrProjectDisabled = errors.New("this project has been disabled for write operations until you re-subscribe your convoy instance") + +func (h *Handler) RequireEnabledProject() func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p, err := h.retrieveProject(r) + if err != nil { + _ = render.Render(w, r, util.NewErrorResponse("failed to retrieve project", http.StatusBadRequest)) + return + } + + if !h.A.Licenser.ProjectEnabled(p.UID) { + _ = render.Render(w, r, util.NewErrorResponse(ErrProjectDisabled.Error(), http.StatusBadRequest)) + return + } + + next.ServeHTTP(w, r) + }) + } +} diff --git a/api/handlers/organisation.go b/api/handlers/organisation.go index d95670f8af..5004d0bf16 100644 --- a/api/handlers/organisation.go +++ b/api/handlers/organisation.go @@ -67,6 +67,7 @@ func (h *Handler) CreateOrganisation(w http.ResponseWriter, r *http.Request) { OrgMemberRepo: postgres.NewOrgMemberRepo(h.A.DB, h.A.Cache), NewOrg: &newOrg, User: user, + Licenser: h.A.Licenser, } organisation, err := co.Run(r.Context()) diff --git a/api/handlers/organisation_invite.go b/api/handlers/organisation_invite.go index b205f10772..2e11effcc1 100644 --- a/api/handlers/organisation_invite.go +++ b/api/handlers/organisation_invite.go @@ -47,6 +47,7 @@ func (h *Handler) InviteUserToOrganisation(w http.ResponseWriter, r *http.Reques Queue: h.A.Queue, InviteRepo: postgres.NewOrgInviteRepo(h.A.DB, h.A.Cache), InviteeEmail: newIV.InviteeEmail, + Licenser: h.A.Licenser, Role: newIV.Role, User: user, Organisation: org, @@ -104,6 +105,7 @@ func (h *Handler) ProcessOrganisationMemberInvite(w http.ResponseWriter, r *http UserRepo: postgres.NewUserRepo(h.A.DB, h.A.Cache), OrgRepo: postgres.NewOrgRepo(h.A.DB, h.A.Cache), OrgMemberRepo: postgres.NewOrgMemberRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, Token: token, Accepted: accepted, NewUser: newUser, diff --git a/api/handlers/project.go b/api/handlers/project.go index 032b7f5bb6..402187a46a 100644 --- a/api/handlers/project.go +++ b/api/handlers/project.go @@ -21,9 +21,8 @@ func createProjectService(h *Handler) (*services.ProjectService, error) { projectService, err := services.NewProjectService( apiKeyRepo, projectRepo, eventRepo, - eventDeliveryRepo, h.A.Cache, + eventDeliveryRepo, h.A.Licenser, h.A.Cache, ) - if err != nil { return nil, err } @@ -78,6 +77,8 @@ func (h *Handler) DeleteProject(w http.ResponseWriter, r *http.Request) { return } + h.A.Licenser.RemoveEnabledProject(project.UID) + _ = render.Render(w, r, util.NewServerResponse("Project deleted successfully", nil, http.StatusOK)) } diff --git a/api/handlers/shim.go b/api/handlers/shim.go deleted file mode 100644 index 2fdf6356d2..0000000000 --- a/api/handlers/shim.go +++ /dev/null @@ -1,67 +0,0 @@ -package handlers - -import ( - "fmt" - "net/http" - "strings" - - "github.com/frain-dev/convoy/auth" - "github.com/frain-dev/convoy/internal/pkg/middleware" - "github.com/frain-dev/convoy/util" - "github.com/go-chi/render" -) - -var redirectRoutes = []string{ - "/api/v1/applications", - "/api/v1/events", - "/api/v1/eventdeliveries", - "/api/v1/security", - "/api/v1/subscriptions", - "/api/v1/sources", -} - -func (h *Handler) RedirectToProjects(w http.ResponseWriter, r *http.Request) { - projectID := r.URL.Query().Get("projectID") - - if util.IsStringEmpty(projectID) { - projectID = r.URL.Query().Get("projectID") - } - - if util.IsStringEmpty(projectID) { - authUser := middleware.GetAuthUserFromContext(r.Context()) - - if authUser.Credential.Type == auth.CredentialTypeAPIKey { - projectID = authUser.Role.Project - } - } - - if util.IsStringEmpty(projectID) { - _ = render.Render(w, r, util.NewErrorResponse("projectID query is missing", http.StatusBadRequest)) - return - } - - rElems := strings.Split(r.URL.Path, "/") - - if !(cap(rElems) > 3) { - _ = render.Render(w, r, util.NewErrorResponse("Invalid path", http.StatusBadRequest)) - return - } - - resourcePrefix := strings.Join(rElems[:4], "/") - - if ok := contains(redirectRoutes, resourcePrefix); ok { - forwardedPath := strings.Join(rElems[3:], "/") - redirectURL := fmt.Sprintf("/api/v1/projects/%s/%s?%s", projectID, forwardedPath, r.URL.RawQuery) - - http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) - } -} - -func contains(sl []string, name string) bool { - for _, value := range sl { - if value == name { - return true - } - } - return false -} diff --git a/api/handlers/subscription.go b/api/handlers/subscription.go index 27a6bf18e1..26a83a9a5f 100644 --- a/api/handlers/subscription.go +++ b/api/handlers/subscription.go @@ -216,6 +216,7 @@ func (h *Handler) CreateSubscription(w http.ResponseWriter, r *http.Request) { SubRepo: postgres.NewSubscriptionRepo(h.A.DB, h.A.Cache), EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), SourceRepo: postgres.NewSourceRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, Project: project, NewSubscription: &sub, } @@ -365,6 +366,7 @@ func (h *Handler) UpdateSubscription(w http.ResponseWriter, r *http.Request) { SubRepo: postgres.NewSubscriptionRepo(h.A.DB, h.A.Cache), EndpointRepo: postgres.NewEndpointRepo(h.A.DB, h.A.Cache), SourceRepo: postgres.NewSourceRepo(h.A.DB, h.A.Cache), + Licenser: h.A.Licenser, ProjectId: project.UID, SubscriptionId: chi.URLParam(r, "subscriptionID"), Update: &update, @@ -400,6 +402,11 @@ func (h *Handler) ToggleSubscriptionStatus(w http.ResponseWriter, r *http.Reques // @Security ApiKeyAuth // @Router /v1/projects/{projectID}/subscriptions/test_filter [post] func (h *Handler) TestSubscriptionFilter(w http.ResponseWriter, r *http.Request) { + if !h.A.Licenser.AdvancedSubscriptions() { + _ = render.Render(w, r, util.NewErrorResponse("your instance does not have access to subscription filters, upgrade to access this feature", http.StatusBadRequest)) + return + } + var test models.TestFilter err := util.ReadJSON(r, &test) if err != nil { @@ -442,6 +449,11 @@ func (h *Handler) TestSubscriptionFilter(w http.ResponseWriter, r *http.Request) // @Security ApiKeyAuth // @Router /v1/projects/{projectID}/subscriptions/test_function [post] func (h *Handler) TestSubscriptionFunction(w http.ResponseWriter, r *http.Request) { + if !h.A.Licenser.Transformations() { + _ = render.Render(w, r, util.NewErrorResponse("your instance does not have access to transformations, upgrade to access this feature", http.StatusBadRequest)) + return + } + var test models.FunctionRequest err := util.ReadJSON(r, &test) if err != nil { diff --git a/api/handlers/user.go b/api/handlers/user.go index 7120594120..b6b33caa2d 100644 --- a/api/handlers/user.go +++ b/api/handlers/user.go @@ -48,8 +48,10 @@ func (h *Handler) RegisterUser(w http.ResponseWriter, r *http.Request) { Queue: h.A.Queue, JWT: jwt.NewJwt(&config.Auth.Jwt, h.A.Cache), ConfigRepo: postgres.NewConfigRepo(h.A.DB), - BaseURL: baseUrl, - Data: &newUser, + Licenser: h.A.Licenser, + + BaseURL: baseUrl, + Data: &newUser, } user, token, err := rs.Run(r.Context()) diff --git a/api/ingest.go b/api/ingest.go index 6cbb90ec69..5af361c2b1 100644 --- a/api/ingest.go +++ b/api/ingest.go @@ -3,15 +3,16 @@ package api import ( "encoding/json" "errors" + "io" + "net/http" "strings" + "time" + + "github.com/frain-dev/convoy/api/handlers" "github.com/frain-dev/convoy/pkg/msgpack" "gopkg.in/guregu/null.v4" - "io" - "net/http" - "time" - "github.com/frain-dev/convoy/internal/pkg/dedup" "github.com/go-chi/chi/v5" "github.com/oklog/ulid/v2" @@ -65,6 +66,11 @@ func (a *ApplicationHandler) IngestEvent(w http.ResponseWriter, r *http.Request) return } + if !a.A.Licenser.ProjectEnabled(project.UID) { + _ = render.Render(w, r, util.NewErrorResponse(handlers.ErrProjectDisabled.Error(), http.StatusBadRequest)) + return + } + if source.Type != datastore.HTTPSource { _ = render.Render(w, r, util.NewErrorResponse("Source type needs to be HTTP", http.StatusBadRequest)) diff --git a/api/ingest_integration_test.go b/api/ingest_integration_test.go index e1aa2cf6ac..09e8f53ac6 100644 --- a/api/ingest_integration_test.go +++ b/api/ingest_integration_test.go @@ -209,6 +209,7 @@ func (i *IngestIntegrationTestSuite) Test_IngestEvent_GoodAPIKey() { // Act. i.Router.ServeHTTP(w, req) + fmt.Println("eee", w.Body.String()) // Assert. require.Equal(i.T(), expectedStatusCode, w.Code) } diff --git a/api/server_suite_test.go b/api/server_suite_test.go index f3a5e25623..ba89a798bb 100644 --- a/api/server_suite_test.go +++ b/api/server_suite_test.go @@ -7,7 +7,6 @@ import ( "bytes" "encoding/json" "fmt" - rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" "io" "math/rand" "net/http" @@ -18,6 +17,9 @@ import ( "testing" "time" + noopLicenser "github.com/frain-dev/convoy/internal/pkg/license/noop" + rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" + ncache "github.com/frain-dev/convoy/cache/noop" "github.com/frain-dev/convoy/api/models" @@ -64,8 +66,10 @@ func getConfig() config.Configuration { return cfg } -var once sync.Once -var pDB *postgres.Postgres +var ( + once sync.Once + pDB *postgres.Postgres +) func getDB() database.Database { once.Do(func() { @@ -125,11 +129,12 @@ func buildServer() *ApplicationHandler { ah, _ := NewApplicationHandler( &types.APIOptions{ - DB: db, - Queue: newQueue, - Logger: logger, - Cache: noopCache, - Rate: r, + DB: db, + Queue: newQueue, + Logger: logger, + Cache: noopCache, + Rate: r, + Licenser: noopLicenser.NewLicenser(), }) _ = ah.RegisterPolicy() diff --git a/api/types/types.go b/api/types/types.go index 539c67289c..6b706f2f14 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -5,6 +5,7 @@ import ( "github.com/frain-dev/convoy/cache" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/queue" @@ -13,11 +14,12 @@ import ( type ContextKey string type APIOptions struct { - FFlag *fflag.FFlag - DB database.Database - Queue queue.Queuer - Logger log.StdLogger - Cache cache.Cache - Authz *authz.Authz - Rate limiter.RateLimiter + FFlag *fflag.FFlag + DB database.Database + Queue queue.Queuer + Logger log.StdLogger + Cache cache.Cache + Authz *authz.Authz + Rate limiter.RateLimiter + Licenser license.Licenser } diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index 933f3090d8..0628cb7f78 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -3,12 +3,13 @@ package agent import ( "context" "fmt" - workerSrv "github.com/frain-dev/convoy/cmd/worker" - "github.com/frain-dev/convoy/util" "os" "os/signal" "time" + workerSrv "github.com/frain-dev/convoy/cmd/worker" + "github.com/frain-dev/convoy/util" + "github.com/frain-dev/convoy/api" "github.com/frain-dev/convoy/api/types" "github.com/frain-dev/convoy/auth/realm_chain" @@ -157,12 +158,13 @@ func startServerComponent(ctx context.Context, a *cli.App) error { evHandler, err := api.NewApplicationHandler( &types.APIOptions{ - FFlag: flag, - DB: a.DB, - Queue: a.Queue, - Logger: lo, - Cache: a.Cache, - Rate: a.Rate, + FFlag: flag, + DB: a.DB, + Queue: a.Queue, + Logger: lo, + Cache: a.Cache, + Rate: a.Rate, + Licenser: a.Licenser, }) if err != nil { return err diff --git a/cmd/bootstrap/bootstrap.go b/cmd/bootstrap/bootstrap.go index 1d4b1f60fa..1c4f75614d 100644 --- a/cmd/bootstrap/bootstrap.go +++ b/cmd/bootstrap/bootstrap.go @@ -33,6 +33,15 @@ func AddBootstrapCommand(a *cli.App) *cobra.Command { "ShouldBootstrap": "false", }, RunE: func(cmd *cobra.Command, args []string) error { + ok, err := a.Licenser.CreateUser(context.Background()) + if err != nil { + return err + } + + if !ok { + return services.ErrUserLimit + } + if format != "json" && format != "human" { return errors.New("unsupported output format") } diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index bd26605045..b2d6f970c1 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -8,6 +8,9 @@ import ( "os" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/internal/pkg/license/keygen" + "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/util" @@ -63,6 +66,17 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ return err } + postgresDB, err := postgres.NewDB(cfg) + if err != nil { + return err + } + + *db = *postgresDB + + if _, ok := skipHook[cmd.Use]; ok { + return nil + } + cfg, err = config.Get() // updated if err != nil { return err @@ -116,13 +130,6 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ return err } - postgresDB, err := postgres.NewDB(cfg) - if err != nil { - return err - } - - *db = *postgresDB - hooks := dbhook.Init() // the order matters here @@ -181,6 +188,18 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ app.Rate = rateLimiter + app.Licenser, err = license.NewLicenser(&license.Config{ + KeyGen: keygen.Config{ + LicenseKey: cfg.LicenseKey, + OrgRepo: postgres.NewOrgRepo(app.DB, app.Cache), + UserRepo: postgres.NewUserRepo(app.DB, app.Cache), + ProjectRepo: projectRepo, + }, + }) + if err != nil { + return err + } + // update config singleton with the instance id if _, ok := skipConfigLoadCmd[cmd.Use]; !ok { configRepo := postgres.NewConfigRepo(app.DB) @@ -202,8 +221,16 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ // these commands don't need to load instance config var skipConfigLoadCmd = map[string]struct{}{ "bootstrap": {}, - "version": {}, - "migrate": {}, +} + +// commands dont need the hooks +var skipHook = map[string]struct{}{ + // migrate commands + "up": {}, + "down": {}, + "create": {}, + + "version": {}, } func PostRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args []string) error { @@ -315,6 +342,14 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) { c := &config.Configuration{} + // CONVOY_LICENSE_KEY + licenseKey, err := cmd.Flags().GetString("license-key") + if err != nil { + return nil, err + } + + c.LicenseKey = licenseKey + // CONVOY_DB_TYPE dbType, err := cmd.Flags().GetString("db-type") if err != nil { @@ -632,14 +667,13 @@ func shouldBootstrap(cmd *cobra.Command) bool { } func ensureDefaultUser(ctx context.Context, a *cli.App) error { - pageable := datastore.Pageable{PerPage: 10, Direction: datastore.Next, NextCursor: datastore.DefaultCursor} userRepo := postgres.NewUserRepo(a.DB, a.Cache) - users, _, err := userRepo.LoadUsersPaged(ctx, pageable) + count, err := userRepo.CountUsers(ctx) if err != nil { - return fmt.Errorf("failed to load users - %w", err) + return fmt.Errorf("failed to count users: %v", err) } - if len(users) > 0 { + if count > 0 { return nil } diff --git a/cmd/ingest/ingest.go b/cmd/ingest/ingest.go index d52cf83d16..334f7bdbd1 100644 --- a/cmd/ingest/ingest.go +++ b/cmd/ingest/ingest.go @@ -3,6 +3,7 @@ package ingest import ( "context" "fmt" + "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/internal/pkg/cli" @@ -111,7 +112,7 @@ func StartIngest(ctx context.Context, a *cli.App, cfg config.Configuration, inte return err } - ingest, err := pubsub.NewIngest(ctx, sourceTable, a.Queue, lo, rateLimiter, host) + ingest, err := pubsub.NewIngest(ctx, sourceTable, a.Queue, lo, rateLimiter, a.Licenser, host) if err != nil { return err } diff --git a/cmd/main.go b/cmd/main.go index 429d2fa412..ff7b1d2475 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -73,9 +73,12 @@ func main() { var maxRetrySeconds uint64 + var licenseKey string + var configFile string c.Flags().StringVar(&configFile, "config", "./convoy.json", "Configuration file for convoy") + c.Flags().StringVar(&licenseKey, "license-key", "", "Convoy license key") // db config c.Flags().StringVar(&dbHost, "db-host", "", "Database Host") diff --git a/cmd/server/server.go b/cmd/server/server.go index f05cbaed78..87d9be3486 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -128,12 +128,13 @@ func startConvoyServer(a *cli.App) error { handler, err := api.NewApplicationHandler( &types.APIOptions{ - FFlag: flag, - DB: a.DB, - Queue: a.Queue, - Logger: lo, - Cache: a.Cache, - Rate: a.Rate, + FFlag: flag, + DB: a.DB, + Queue: a.Queue, + Logger: lo, + Cache: a.Cache, + Rate: a.Rate, + Licenser: a.Licenser, }) if err != nil { return err diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index 20c3eac01e..bdbf1071ec 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -3,6 +3,8 @@ package worker import ( "context" "fmt" + "net/http" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" @@ -27,7 +29,6 @@ import ( "github.com/go-chi/render" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" - "net/http" ) func AddWorkerCommand(a *cli.App) *cobra.Command { @@ -237,7 +238,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte telemetry.OptionBackend(pb), telemetry.OptionBackend(mb)) - dispatcher, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, false) + dispatcher, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, a.Licenser, false) if err != nil { a.Logger.WithError(err).Fatal("Failed to create new net dispatcher") return err @@ -246,6 +247,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte consumer.RegisterHandlers(convoy.EventProcessor, task.ProcessEventDelivery( endpointRepo, eventDeliveryRepo, + a.Licenser, projectRepo, a.Queue, rateLimiter, @@ -260,7 +262,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte eventDeliveryRepo, a.Queue, subRepo, - deviceRepo), newTelemetry) + deviceRepo, a.Licenser), newTelemetry) consumer.RegisterHandlers(convoy.RetryEventProcessor, task.ProcessRetryEventDelivery( endpointRepo, @@ -280,6 +282,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte a.Queue, subRepo, deviceRepo, + a.Licenser, subscriptionsTable), newTelemetry) consumer.RegisterHandlers(convoy.CreateDynamicEventProcessor, task.ProcessDynamicEventCreation( @@ -289,7 +292,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte eventDeliveryRepo, a.Queue, subRepo, - deviceRepo), newTelemetry) + deviceRepo, a.Licenser), newTelemetry) consumer.RegisterHandlers(convoy.RetentionPolicies, task.RetentionPolicies( configRepo, @@ -317,7 +320,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte } consumer.RegisterHandlers(convoy.NotificationProcessor, task.ProcessNotifications(sc), nil) - consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo), nil) + consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher), nil) consumer.RegisterHandlers(convoy.DeleteArchivedTasksProcessor, task.DeleteArchivedTasks(a.Queue, rd), nil) metrics.RegisterQueueMetrics(a.Queue, a.DB) diff --git a/config/config.go b/config/config.go index 6e9b125ea7..8c387a3c16 100644 --- a/config/config.go +++ b/config/config.go @@ -365,6 +365,7 @@ type Configuration struct { InstanceIngestRate int `json:"instance_ingest_rate" envconfig:"CONVOY_INSTANCE_INGEST_RATE"` WorkerExecutionMode ExecutionMode `json:"worker_execution_mode" envconfig:"CONVOY_WORKER_EXECUTION_MODE"` MaxRetrySeconds uint64 `json:"max_retry_seconds,omitempty" envconfig:"CONVOY_MAX_RETRY_SECONDS"` + LicenseKey string `json:"license_key" envconfig:"CONVOY_LICENSE_KEY"` } type PyroscopeConfiguration struct { diff --git a/database/postgres/organisation.go b/database/postgres/organisation.go index d35881b913..14a13b771a 100644 --- a/database/postgres/organisation.go +++ b/database/postgres/organisation.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/cache" ncache "github.com/frain-dev/convoy/cache/noop" @@ -79,6 +80,11 @@ const ( GROUP BY id ORDER BY id DESC LIMIT 1` + + countOrganizations = ` + SELECT COUNT(*) AS count + FROM convoy.organisations + WHERE deleted_at IS NULL` ) type orgRepo struct { @@ -254,6 +260,16 @@ func (o *orgRepo) DeleteOrganisation(ctx context.Context, uid string) error { return nil } +func (o *orgRepo) CountOrganisations(ctx context.Context) (int64, error) { + var count int64 + err := o.db.GetContext(ctx, &count, countOrganizations) + if err != nil { + return 0, err + } + + return count, nil +} + func (o *orgRepo) FetchOrganisationByID(ctx context.Context, id string) (*datastore.Organisation, error) { fromCache, err := o.readFromCache(ctx, id, func() (*datastore.Organisation, error) { org := &datastore.Organisation{} @@ -267,7 +283,6 @@ func (o *orgRepo) FetchOrganisationByID(ctx context.Context, id string) (*datast return org, nil }) - if err != nil { return nil, err } @@ -288,7 +303,6 @@ func (o *orgRepo) FetchOrganisationByAssignedDomain(ctx context.Context, domain return org, nil }) - if err != nil { return nil, err } @@ -309,7 +323,6 @@ func (o *orgRepo) FetchOrganisationByCustomDomain(ctx context.Context, domain st return org, nil }) - if err != nil { return nil, err } diff --git a/database/postgres/organisation_member.go b/database/postgres/organisation_member.go index 2c5670d207..259c91477c 100644 --- a/database/postgres/organisation_member.go +++ b/database/postgres/organisation_member.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/frain-dev/convoy/cache" "github.com/frain-dev/convoy/util" diff --git a/database/postgres/organisation_test.go b/database/postgres/organisation_test.go index 26a9c2ac15..4cbdaae797 100644 --- a/database/postgres/organisation_test.go +++ b/database/postgres/organisation_test.go @@ -47,6 +47,33 @@ func TestLoadOrganisationsPaged(t *testing.T) { require.Equal(t, 2, len(organisations)) } +func TestCountOrganisations(t *testing.T) { + db, closeFn := getDB(t) + defer closeFn() + + orgRepo := NewOrgRepo(db, nil) + + user := seedUser(t, db) + count := 10 + for i := 0; i < count; i++ { + org := &datastore.Organisation{ + UID: ulid.Make().String(), + OwnerID: user.UID, + Name: fmt.Sprintf("org%d", i), + CustomDomain: null.NewString(ulid.Make().String(), true), + AssignedDomain: null.NewString(ulid.Make().String(), true), + } + + err := orgRepo.CreateOrganisation(context.Background(), org) + require.NoError(t, err) + } + + orgCount, err := orgRepo.CountOrganisations(context.Background()) + + require.NoError(t, err) + require.Equal(t, int64(count), orgCount) +} + func TestCreateOrganisation(t *testing.T) { db, closeFn := getDB(t) defer closeFn() diff --git a/database/postgres/project.go b/database/postgres/project.go index 92e87ebaea..8c8d860ea9 100644 --- a/database/postgres/project.go +++ b/database/postgres/project.go @@ -111,8 +111,7 @@ const ( LEFT JOIN convoy.project_configurations c ON p.project_configuration_id = c.id WHERE p.id = $1 AND p.deleted_at IS NULL; - ` - +` fetchProjects = ` SELECT p.id, @@ -210,6 +209,11 @@ const ( GROUP BY p.id ORDER BY events_count DESC; ` + + countProjects = ` + SELECT COUNT(*) AS count + FROM convoy.projects + WHERE deleted_at IS NULL` ) type projectRepo struct { @@ -225,6 +229,16 @@ func NewProjectRepo(db database.Database, ca cache.Cache) datastore.ProjectRepos return &projectRepo{db: db.GetDB(), hook: db.GetHook(), cache: ca} } +func (o *projectRepo) CountProjects(ctx context.Context) (int64, error) { + var count int64 + err := o.db.GetContext(ctx, &count, countProjects) + if err != nil { + return 0, err + } + + return count, nil +} + func (p *projectRepo) CreateProject(ctx context.Context, project *datastore.Project) error { tx, err := p.db.BeginTxx(ctx, &sql.TxOptions{}) if err != nil { diff --git a/database/postgres/project_test.go b/database/postgres/project_test.go index 67c31a4ed1..64278e2b64 100644 --- a/database/postgres/project_test.go +++ b/database/postgres/project_test.go @@ -61,6 +61,32 @@ func Test_FetchProjectByID(t *testing.T) { require.Equal(t, newProject, dbProject) } +func TestCountProjects(t *testing.T) { + db, closeFn := getDB(t) + defer closeFn() + + projectRepository := NewProjectRepo(db, nil) + org := seedOrg(t, db) + + count := 10 + for i := 0; i < count; i++ { + project := &datastore.Project{ + UID: ulid.Make().String(), + Name: ulid.Make().String(), + OrganisationID: org.UID, + Type: datastore.IncomingProject, + Config: &datastore.DefaultProjectConfig, + } + + err := projectRepository.CreateProject(context.Background(), project) + require.NoError(t, err) + } + + projectCount, err := projectRepository.CountProjects(context.Background()) + require.NoError(t, err) + require.Equal(t, int64(count), projectCount) +} + func Test_CreateProject(t *testing.T) { db, closeFn := getDB(t) defer closeFn() diff --git a/database/postgres/users.go b/database/postgres/users.go index e18b36b1a4..f4e89eb450 100644 --- a/database/postgres/users.go +++ b/database/postgres/users.go @@ -41,35 +41,10 @@ const ( WHERE deleted_at IS NULL ` - fetchUsersPaginated = ` - SELECT * FROM convoy.users WHERE deleted_at IS NULL` - - fetchUsersPagedForward = ` - %s - AND id <= :cursor - GROUP BY id - ORDER BY id DESC - LIMIT :limit` - - fetchUsersPagedBackward = ` - WITH users AS ( - %s - AND id >= :cursor - GROUP BY id - ORDER BY id ASC - LIMIT :limit - ) - - SELECT * FROM users ORDER BY id DESC` - - countPrevUsers = ` - SELECT COUNT(DISTINCT(id)) AS count + countUsers = ` + SELECT COUNT(*) AS count FROM convoy.users - WHERE deleted_at IS NULL - AND id > :cursor - GROUP BY id - ORDER BY id DESC - LIMIT 1` + WHERE deleted_at IS NULL` ) var ( @@ -191,91 +166,12 @@ func (u *userRepo) FindUserByEmailVerificationToken(ctx context.Context, token s return user, nil } -func (u *userRepo) LoadUsersPaged(ctx context.Context, pageable datastore.Pageable) ([]datastore.User, datastore.PaginationData, error) { - arg := map[string]interface{}{ - "limit": pageable.Limit(), - "cursor": pageable.Cursor(), - } - - var query string - if pageable.Direction == datastore.Next { - query = fetchUsersPagedForward - } else { - query = fetchUsersPagedBackward - } - - query = fmt.Sprintf(query, fetchUsersPaginated) - - query, args, err := sqlx.Named(query, arg) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - query, args, err = sqlx.In(query, args...) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - query = u.db.Rebind(query) - - rows, err := u.db.QueryxContext(ctx, query, args...) +func (o *userRepo) CountUsers(ctx context.Context) (int64, error) { + var count int64 + err := o.db.GetContext(ctx, &count, countUsers) if err != nil { - return nil, datastore.PaginationData{}, err + return 0, err } - defer closeWithError(rows) - - var users []datastore.User - for rows.Next() { - var user datastore.User - err = rows.StructScan(&user) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - users = append(users, user) - } - - var count datastore.PrevRowCount - if len(users) > 0 { - var countQuery string - var qargs []interface{} - first := users[0] - qarg := arg - qarg["cursor"] = first.UID - - countQuery, qargs, err = sqlx.Named(countPrevUsers, qarg) - if err != nil { - return nil, datastore.PaginationData{}, err - } - - countQuery = u.db.Rebind(countQuery) - - // count the row number before the first row - rows, err := u.db.QueryxContext(ctx, countQuery, qargs...) - if err != nil { - return nil, datastore.PaginationData{}, err - } - defer closeWithError(rows) - - if rows.Next() { - err = rows.StructScan(&count) - if err != nil { - return nil, datastore.PaginationData{}, err - } - } - } - - ids := make([]string, len(users)) - for i := range users { - ids[i] = users[i].UID - } - - if len(users) > pageable.PerPage { - users = users[:len(users)-1] - } - - pagination := &datastore.PaginationData{PrevRowCount: count} - pagination = pagination.Build(pageable, ids) - return users, *pagination, nil + return count, nil } diff --git a/database/postgres/users_test.go b/database/postgres/users_test.go index a020ea6d5c..276bad6f99 100644 --- a/database/postgres/users_test.go +++ b/database/postgres/users_test.go @@ -87,6 +87,31 @@ func Test_CreateUser(t *testing.T) { } } +func TestCountUsers(t *testing.T) { + db, closeFn := getDB(t) + defer closeFn() + + userRepo := NewUserRepo(db, nil) + count := 10 + + for i := 0; i < count; i++ { + u := &datastore.User{ + UID: ulid.Make().String(), + FirstName: "test", + LastName: "test", + Email: fmt.Sprintf("%s@test.com", ulid.Make().String()), + } + + err := userRepo.CreateUser(context.Background(), u) + require.NoError(t, err) + } + + userCount, err := userRepo.CountUsers(context.Background()) + + require.NoError(t, err) + require.Equal(t, int64(count), userCount) +} + func Test_FindUserByEmail(t *testing.T) { db, closeFn := getDB(t) defer closeFn() @@ -210,76 +235,6 @@ func Test_FindUserByEmailVerificationToken(t *testing.T) { require.Equal(t, user, newUser) } -func Test_LoadUsersPaged(t *testing.T) { - type Expected struct { - paginationData datastore.PaginationData - } - - tests := []struct { - name string - pageData datastore.Pageable - count int - expected Expected - }{ - { - name: "Load Users Paged - 10 records", - pageData: datastore.Pageable{PerPage: 3}, - count: 10, - expected: Expected{ - paginationData: datastore.PaginationData{ - PerPage: 3, - }, - }, - }, - - { - name: "Load Users Paged - 12 records", - pageData: datastore.Pageable{PerPage: 4}, - count: 12, - expected: Expected{ - paginationData: datastore.PaginationData{ - PerPage: 4, - }, - }, - }, - - { - name: "Load Users Paged - 5 records", - pageData: datastore.Pageable{PerPage: 3}, - count: 5, - expected: Expected{ - paginationData: datastore.PaginationData{ - PerPage: 3, - }, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - db, closeFn := getDB(t) - defer closeFn() - - userRepo := NewUserRepo(db, nil) - - for i := 0; i < tc.count; i++ { - user := &datastore.User{ - UID: ulid.Make().String(), - FirstName: "test", - LastName: "test", - Email: fmt.Sprintf("%s@test.com", ulid.Make().String()), - } - require.NoError(t, userRepo.CreateUser(context.Background(), user)) - } - - _, pageable, err := userRepo.LoadUsersPaged(context.Background(), tc.pageData) - - require.NoError(t, err) - require.Equal(t, tc.expected.paginationData.PerPage, pageable.PerPage) - }) - } -} - func Test_UpdateUser(t *testing.T) { db, closeFn := getDB(t) defer closeFn() diff --git a/datastore/repository.go b/datastore/repository.go index cad2329a89..f150cd5a6a 100644 --- a/datastore/repository.go +++ b/datastore/repository.go @@ -57,6 +57,7 @@ type EventRepository interface { type ProjectRepository interface { LoadProjects(context.Context, *ProjectFilter) ([]*Project, error) CreateProject(context.Context, *Project) error + CountProjects(ctx context.Context) (int64, error) UpdateProject(context.Context, *Project) error DeleteProject(ctx context.Context, uid string) error FetchProjectByID(context.Context, string) (*Project, error) @@ -66,6 +67,7 @@ type ProjectRepository interface { type OrganisationRepository interface { LoadOrganisationsPaged(context.Context, Pageable) ([]Organisation, PaginationData, error) + CountOrganisations(ctx context.Context) (int64, error) CreateOrganisation(context.Context, *Organisation) error UpdateOrganisation(context.Context, *Organisation) error DeleteOrganisation(context.Context, string) error @@ -163,11 +165,11 @@ type JobRepository interface { type UserRepository interface { CreateUser(context.Context, *User) error UpdateUser(ctx context.Context, user *User) error + CountUsers(ctx context.Context) (int64, error) FindUserByEmail(context.Context, string) (*User, error) FindUserByID(context.Context, string) (*User, error) FindUserByToken(context.Context, string) (*User, error) FindUserByEmailVerificationToken(ctx context.Context, token string) (*User, error) - LoadUsersPaged(context.Context, Pageable) ([]User, PaginationData, error) } type ConfigurationRepository interface { diff --git a/docs/docs.go b/docs/docs.go index 63f96d0ee0..1ba132971a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1,4 +1,4 @@ -// Package docs Code generated by swaggo/swag at 2024-07-10 21:43:37.804513 -0700 PDT m=+1.709850626. DO NOT EDIT +// Package docs Code generated by swaggo/swag at 2024-08-30 12:08:22.812424 +0100 BST m=+2.397263876. DO NOT EDIT package docs import "github.com/swaggo/swag" @@ -5560,6 +5560,9 @@ const docTemplate = `{ "msg_id": { "type": "string" }, + "project_id": { + "type": "string" + }, "request_http_header": { "$ref": "#/definitions/datastore.HttpHeader" }, diff --git a/docs/swagger.json b/docs/swagger.json index 6464c1fc5b..8edd39894e 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5557,6 +5557,9 @@ "msg_id": { "type": "string" }, + "project_id": { + "type": "string" + }, "request_http_header": { "$ref": "#/definitions/datastore.HttpHeader" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 65cf9eb72b..7d69809a59 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -83,6 +83,8 @@ definitions: type: string msg_id: type: string + project_id: + type: string request_http_header: $ref: '#/definitions/datastore.HttpHeader' response_data: diff --git a/docs/v3/openapi3.json b/docs/v3/openapi3.json index 4c36320f73..53f51e1aba 100644 --- a/docs/v3/openapi3.json +++ b/docs/v3/openapi3.json @@ -128,6 +128,9 @@ "msg_id": { "type": "string" }, + "project_id": { + "type": "string" + }, "request_http_header": { "$ref": "#/components/schemas/datastore.HttpHeader" }, diff --git a/docs/v3/openapi3.yaml b/docs/v3/openapi3.yaml index a8615506f9..98a7b85177 100644 --- a/docs/v3/openapi3.yaml +++ b/docs/v3/openapi3.yaml @@ -83,6 +83,8 @@ components: type: string msg_id: type: string + project_id: + type: string request_http_header: $ref: '#/components/schemas/datastore.HttpHeader' response_data: diff --git a/ee/cmd/server/server.go b/ee/cmd/server/server.go index 0fe53a6e6f..3078cc19d5 100644 --- a/ee/cmd/server/server.go +++ b/ee/cmd/server/server.go @@ -2,9 +2,10 @@ package server import ( "errors" - "github.com/frain-dev/convoy/internal/pkg/fflag" "time" + "github.com/frain-dev/convoy/internal/pkg/fflag" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/worker" @@ -157,12 +158,13 @@ func StartConvoyServer(a *cli.App) error { handler, err := api.NewEHandler( &types.APIOptions{ - FFlag: flag, - DB: a.DB, - Queue: a.Queue, - Logger: lo, - Cache: a.Cache, - Rate: a.Rate, + FFlag: flag, + DB: a.DB, + Queue: a.Queue, + Logger: lo, + Cache: a.Cache, + Rate: a.Rate, + Licenser: a.Licenser, }) if err != nil { return err diff --git a/generate.go b/generate.go index 93e3d71b3a..191add2520 100644 --- a/generate.go +++ b/generate.go @@ -10,3 +10,4 @@ package convoy //go:generate mockgen --source internal/pkg/pubsub/pubsub.go --destination mocks/pubsub.go -package mocks //go:generate mockgen --source internal/pkg/dedup/dedup.go --destination mocks/dedup.go -package mocks //go:generate mockgen --source internal/pkg/memorystore/table.go --destination mocks/table.go -package mocks +//go:generate mockgen --source internal/pkg/license/license.go --destination mocks/license.go -package mocks diff --git a/go.mod b/go.mod index 75e9afb1ef..56257ffc23 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/frain-dev/convoy -go 1.21 +go 1.21.0 + +toolchain go1.22.3 require ( cloud.google.com/go/pubsub v1.33.0 @@ -9,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.44.327 github.com/danvixent/asynqmon v0.7.3 github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 - github.com/docker/compose/v2 v2.28.1 + github.com/docker/compose/v2 v2.29.1 github.com/dop251/goja v0.0.0-20231014103939-873a1496dc8e github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562 github.com/exaring/otelpgx v0.5.4 @@ -35,7 +37,8 @@ require ( github.com/jaswdr/faker v1.10.2 github.com/jmoiron/sqlx v1.3.5 github.com/kelseyhightower/envconfig v1.4.0 - github.com/lib/pq v1.10.9 + github.com/keygen-sh/keygen-go/v3 v3.2.0 + github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 github.com/mixpanel/mixpanel-go v1.2.1 github.com/newrelic/go-agent/v3 v3.20.4 @@ -68,8 +71,8 @@ require ( go.opentelemetry.io/otel/sdk v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.22.0 - google.golang.org/api v0.128.0 + golang.org/x/crypto v0.23.0 + google.golang.org/api v0.149.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/guregu/null.v4 v4.0.0 ) @@ -81,7 +84,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.11.5 // indirect + github.com/Microsoft/hcsshim v0.11.7 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect @@ -99,9 +102,10 @@ require ( github.com/aws/smithy-go v1.19.0 // indirect github.com/buger/goterm v1.0.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/compose-spec/compose-go/v2 v2.1.3 // indirect + github.com/compose-spec/compose-go/v2 v2.1.5 // indirect github.com/containerd/console v1.0.4 // indirect - github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/containerd v1.7.20 // indirect + github.com/containerd/containerd/api v1.7.19 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -111,11 +115,12 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect - github.com/docker/buildx v0.15.1 // indirect - github.com/docker/cli v27.0.3+incompatible // indirect + github.com/docker/buildx v0.16.0 // indirect + github.com/docker/cli v27.1.0+incompatible // indirect + github.com/docker/cli-docs-tool v0.8.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v27.0.3+incompatible // indirect - github.com/docker/docker-credential-helpers v0.8.0 // indirect + github.com/docker/docker v27.1.0+incompatible // indirect + github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect @@ -130,7 +135,7 @@ require ( github.com/go-redis/redis/v7 v7.4.1 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-viper/mapstructure/v2 v2.0.0 // indirect - github.com/gofrs/flock v0.8.1 // indirect + github.com/gofrs/flock v0.12.0 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect @@ -138,8 +143,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect - github.com/google/s2a-go v0.1.5 // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -156,6 +161,8 @@ require ( github.com/jonboulle/clockwork v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/keygen-sh/go-update v1.0.0 // indirect + github.com/keygen-sh/jsonapi-go v1.2.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -164,8 +171,9 @@ require ( github.com/mattn/go-shellwords v1.0.12 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/buildkit v0.14.1 // indirect + github.com/moby/buildkit v0.15.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect @@ -182,6 +190,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -204,6 +213,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -217,16 +227,16 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect - golang.org/x/term v0.19.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect + golang.org/x/term v0.20.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/api v0.29.2 // indirect @@ -242,10 +252,9 @@ require ( ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute v1.23.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go v0.110.10 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -259,20 +268,20 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sergi/go-diff v1.0.0 // indirect github.com/spf13/cast v1.5.1 // indirect @@ -280,16 +289,15 @@ require ( github.com/vmihailenco/msgpack/v5 v5.3.5 github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/grpc v1.59.0 + google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect + google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index cfeca4f84d..398e2ebb03 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= @@ -112,12 +112,10 @@ cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQH cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= @@ -198,8 +196,8 @@ cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= @@ -209,8 +207,8 @@ cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0 cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= -cloud.google.com/go/kms v1.15.3 h1:RYsbxTRmk91ydKCzekI2YjryO4c5Y2M80Zwcs9/D/cI= -cloud.google.com/go/kms v1.15.3/go.mod h1:AJdXqHxS2GlPyduM99s9iGqi2nwbviBbhV/hdmt4iOQ= +cloud.google.com/go/kms v1.15.5 h1:pj1sRfut2eRbD9pFRjNnPNg/CzJPuQAzUujMIM1vVeM= +cloud.google.com/go/kms v1.15.5/go.mod h1:cU2H5jnp6G2TDpUGZyqTCoy1n16fbubHZjmVXSMtwDI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= @@ -435,8 +433,8 @@ github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+V github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim v0.9.1/go.mod h1:Y/0uV2jUab5kBI7SQgl62at0AVX7uaruzADAVmxm3eM= -github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38= -github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -601,8 +599,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= -github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi1KW7R5esrLE= -github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= +github.com/compose-spec/compose-go/v2 v2.1.5 h1:6YoC9ik3NXdSYtgRn51EMZ2DxzGPyGjZ8M0B7mXTXeQ= +github.com/compose-spec/compose-go/v2 v2.1.5/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -643,8 +641,10 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= -github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/containerd v1.7.20 h1:Sl6jQYk3TRavaU83h66QMbI2Nqg9Jm6qzwX57Vsn1SQ= +github.com/containerd/containerd v1.7.20/go.mod h1:52GsS5CwquuqPuLncsXwG0t2CiUce+KsNHJZQJvAgR0= +github.com/containerd/containerd/api v1.7.19 h1:VWbJL+8Ap4Ju2mx9c9qS1uFSB1OVYr5JJrW2yT5vFoA= +github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -758,6 +758,8 @@ github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/deepmap/oapi-codegen v1.9.0/go.mod h1:7t4DbSxmAffcTEgrWvsPYEE2aOARZ8ZKWp3hDuZkHNc= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= @@ -773,13 +775,15 @@ github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnm github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/buildx v0.15.1 h1:1cO6JIc0rOoC8tlxfXoh1HH1uxaNvYH1q7J7kv5enhw= -github.com/docker/buildx v0.15.1/go.mod h1:16DQgJqoggmadc1UhLaUTPqKtR+PlByN/kyXFdkhFCo= +github.com/docker/buildx v0.16.0 h1:LurEflyb6BBoLtDwJY1dw9dLHKzEgGvCjAz67QI0xO0= +github.com/docker/buildx v0.16.0/go.mod h1:4xduW7BOJ2B11AyORKZFDKjF6Vcb4EgTYnV2nunxv9I= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v27.0.3+incompatible h1:usGs0/BoBW8MWxGeEtqPMkzOY56jZ6kYlSN5BLDioCQ= -github.com/docker/cli v27.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/compose/v2 v2.28.1 h1:ORPfiVHrpnRQBDoC3F8JJyWAY8N5gWuo3FgwyivxFdM= -github.com/docker/compose/v2 v2.28.1/go.mod h1:wDtGQFHe99sPLCHXeVbCkc+Wsl4Y/2ZxiAJa/nga6rA= +github.com/docker/cli v27.1.0+incompatible h1:P0KSYmPtNbmx59wHZvG6+rjivhKDRA1BvvWM0f5DgHc= +github.com/docker/cli v27.1.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli-docs-tool v0.8.0 h1:YcDWl7rQJC3lJ7WVZRwSs3bc9nka97QLWfyJQli8yJU= +github.com/docker/cli-docs-tool v0.8.0/go.mod h1:8TQQ3E7mOXoYUs811LiPdUnAhXrcVsBIrW21a5pUbdk= +github.com/docker/compose/v2 v2.29.1 h1:H8oyStDcpjFvyczQ5IuKIE+Bz9jEh8NsRhrAFFlH90U= +github.com/docker/compose/v2 v2.29.1/go.mod h1:oxZ/omiQ4pHlVtp4dJ/B/fZ1pCDlBvYt97UnVelbNKk= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -788,11 +792,11 @@ github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE= -github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= +github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= -github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -1007,8 +1011,8 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.24.2/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/flock v0.12.0 h1:xHW8t8GPAiGtqz7KxiSqfOEXwpOaqhpYZrTE2MQBgXY= +github.com/gofrs/flock v0.12.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -1120,11 +1124,11 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= -github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1137,8 +1141,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -1200,6 +1204,8 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1: github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -1311,6 +1317,12 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/keygen-sh/go-update v1.0.0 h1:M65sTVUHUO07tEK4l1Hq7u5D4kdEqkcgfdzUt3q3S08= +github.com/keygen-sh/go-update v1.0.0/go.mod h1:wn0UWRHLnBP5hwXtj1IdHZqWlHvIadh2Nn+becFf8Ro= +github.com/keygen-sh/jsonapi-go v1.2.1 h1:NTSIAxl2+7S5fPnKgrYwNjQSWbdKRtrFq26SD8AOkiU= +github.com/keygen-sh/jsonapi-go v1.2.1/go.mod h1:8j9vsLiKyJyDqmt8r3tYaYNmXszq2+cFhoO6QdMdAes= +github.com/keygen-sh/keygen-go/v3 v3.2.0 h1:OJqnGtY6z4ZA434kZqfNVHDmSrN5zq4l4XItcB3tECY= +github.com/keygen-sh/keygen-go/v3 v3.2.0/go.mod h1:YoFyryzXEk6XrbT3H8EUUU+JcIJkQu414TA6CvZgS/E= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1321,8 +1333,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1355,9 +1367,8 @@ github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmt github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= @@ -1438,6 +1449,8 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -1450,8 +1463,8 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mixpanel/mixpanel-go v1.2.1 h1:iykbHKomTJjVoWU95Vt1sjZy4HLt8UOYacMEEEMFBok= github.com/mixpanel/mixpanel-go v1.2.1/go.mod h1:mPGaNhBoZMJuLu8k7Y1KhU5n8Vw13rxQZZjHj+b9RLk= -github.com/moby/buildkit v0.14.1 h1:2epLCZTkn4CikdImtsLtIa++7DzCimrrZCT1sway+oI= -github.com/moby/buildkit v0.14.1/go.mod h1:1XssG7cAqv5Bz1xcGMxJL123iCv5TYN4Z/qf647gfuk= +github.com/moby/buildkit v0.15.0 h1:vnZLThPr9JU6SvItctKoa6NfgPZ8oUApg/TCOaa/SVs= +github.com/moby/buildkit v0.15.0/go.mod h1:oN9S+8I7wF26vrqn9NuAF6dFSyGTfXvtiu9o1NlnnH4= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -1521,6 +1534,8 @@ github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94 h1:YXfl+eCNmAQhVbSNQ85bSi1n4qhUBPW8Qq9Rac4pt/s= +github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94/go.mod h1:WUcXjUd98qaCVFb6j8Xc87MsKeMCXDu9Nk8JRJ9SeC8= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -1601,8 +1616,8 @@ github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.m github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= @@ -1702,8 +1717,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/r3labs/diff/v3 v3.0.1 h1:CBKqf3XmNRHXKmdU7mZP1w7TV0pDyVCis1AUHtA4Xtg= github.com/r3labs/diff/v3 v3.0.1/go.mod h1:f1S9bourRbiM66NskseyUdo0fTmEE0qKrikYJX63dgo= @@ -1888,6 +1903,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c h1:+6wg/4ORAbnSoGDzg2Q1i3CeMcT/jjhye/ZfnBHy7/M= github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= @@ -1995,8 +2012,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= @@ -2065,14 +2082,14 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -2183,7 +2200,6 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -2205,8 +2221,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2233,8 +2249,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2406,8 +2422,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2425,8 +2441,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2582,8 +2598,8 @@ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= -google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= -google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2592,8 +2608,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -2704,12 +2718,12 @@ google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnp google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= +google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY= +google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2754,8 +2768,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/pkg/cli/cli.go b/internal/pkg/cli/cli.go index 143ea6406b..31e1894c2a 100644 --- a/internal/pkg/cli/cli.go +++ b/internal/pkg/cli/cli.go @@ -2,6 +2,8 @@ package cli import ( "context" + + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/cache" @@ -14,12 +16,13 @@ import ( // App is the core dependency of the entire binary. type App struct { - Version string - DB database.Database - Queue queue.Queuer - Logger log.StdLogger - Cache cache.Cache - Rate limiter.RateLimiter + Version string + DB database.Database + Queue queue.Queuer + Logger log.StdLogger + Cache cache.Cache + Rate limiter.RateLimiter + Licenser license.Licenser // TODO(subomi): Let's make this cleaner. TracerShutdown func(context.Context) error diff --git a/internal/pkg/license/keygen/community.go b/internal/pkg/license/keygen/community.go new file mode 100644 index 0000000000..7b8322599a --- /dev/null +++ b/internal/pkg/license/keygen/community.go @@ -0,0 +1,55 @@ +package keygen + +import ( + "context" + + "github.com/keygen-sh/keygen-go/v3" + + "github.com/frain-dev/convoy/datastore" +) + +const ( + projectLimit = 2 + orgLimit = 1 + userLimit = 1 +) + +func communityLicenser(ctx context.Context, orgRepo datastore.OrganisationRepository, userRepo datastore.UserRepository, projectRepo datastore.ProjectRepository) (*Licenser, error) { + m, err := enforceProjectLimit(ctx, projectRepo) + if err != nil { + return nil, err + } + + return &Licenser{ + planType: CommunityPlan, + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: orgLimit}, + CreateUser: {Limit: userLimit}, + CreateProject: {Limit: projectLimit}, + }, + license: &keygen.License{}, + enabledProjects: m, + orgRepo: orgRepo, + userRepo: userRepo, + projectRepo: projectRepo, + }, nil +} + +func enforceProjectLimit(ctx context.Context, projectRepo datastore.ProjectRepository) (map[string]bool, error) { + projects, err := projectRepo.LoadProjects(ctx, &datastore.ProjectFilter{}) + if err != nil { + return nil, err + } + + if len(projects) > projectLimit { + // enabled projects are not within accepted count, allow only the last projects to be active + projects = projects[len(projects)-projectLimit:] + } + + m := map[string]bool{} + for _, p := range projects { + m[p.UID] = true + } + + return m, nil +} diff --git a/internal/pkg/license/keygen/community_test.go b/internal/pkg/license/keygen/community_test.go new file mode 100644 index 0000000000..c5244a8f7d --- /dev/null +++ b/internal/pkg/license/keygen/community_test.go @@ -0,0 +1,89 @@ +package keygen + +import ( + "context" + "testing" + + "github.com/frain-dev/convoy/datastore" + + "github.com/stretchr/testify/require" + + "github.com/frain-dev/convoy/mocks" + "go.uber.org/mock/gomock" +) + +func Test_communityLicenser(t *testing.T) { + testCases := []struct { + name string + featureList map[Feature]*Properties + expectedFeatureList map[Feature]*Properties + expectedEnabledProjects map[string]bool + dbFn func(projectRepo datastore.ProjectRepository) + }{ + { + name: "should_disable_projects", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + dbFn: func(projectRepo datastore.ProjectRepository) { + pr, _ := projectRepo.(*mocks.MockProjectRepository) + pr.EXPECT().LoadProjects(gomock.Any(), gomock.Any()).Times(1).Return([]*datastore.Project{{UID: "01111111"}, {UID: "02222"}, {UID: "033333"}, {UID: "044444"}}, nil) + }, + expectedFeatureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + expectedEnabledProjects: map[string]bool{ + "033333": true, + "044444": true, + }, + }, + { + name: "should_not_disable_projects", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + dbFn: func(projectRepo datastore.ProjectRepository) { + pr, _ := projectRepo.(*mocks.MockProjectRepository) + pr.EXPECT().LoadProjects(gomock.Any(), gomock.Any()).Times(1).Return([]*datastore.Project{{UID: "033333"}, {UID: "044444"}}, nil) + }, + expectedEnabledProjects: map[string]bool{ + "033333": true, + "044444": true, + }, + expectedFeatureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateUser: {Limit: 1}, + CreateProject: {Limit: 2}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + orgRepo := mocks.NewMockOrganisationRepository(ctrl) + userRepository := mocks.NewMockUserRepository(ctrl) + projectRepo := mocks.NewMockProjectRepository(ctrl) + + if tc.dbFn != nil { + tc.dbFn(projectRepo) + } + + l, err := communityLicenser(context.Background(), orgRepo, userRepository, projectRepo) + require.NoError(t, err) + + require.Equal(t, tc.expectedFeatureList, l.featureList) + require.Equal(t, tc.expectedEnabledProjects, l.enabledProjects) + require.Equal(t, orgRepo, l.orgRepo) + require.Equal(t, userRepository, l.userRepo) + require.Equal(t, projectRepo, l.projectRepo) + }) + } +} diff --git a/internal/pkg/license/keygen/feature.go b/internal/pkg/license/keygen/feature.go new file mode 100644 index 0000000000..4e13d73c66 --- /dev/null +++ b/internal/pkg/license/keygen/feature.go @@ -0,0 +1,44 @@ +package keygen + +type ( + Feature string + PlanType string +) + +const ( + CreateOrg Feature = "CREATE_ORG" + CreateUser Feature = "CREATE_USER" + CreateProject Feature = "CREATE_PROJECT" + UseForwardProxy Feature = "USE_FORWARD_PROXY" + ExportPrometheusMetrics Feature = "EXPORT_PROMETHEUS_METRICS" + AdvancedEndpointMgmt Feature = "ADVANCED_ENDPOINT_MANAGEMENT" + AdvancedWebhookArchiving Feature = "ADVANCED_WEBHOOK_ARCHIVING" + AdvancedMsgBroker Feature = "ADVANCED_MESSAGE_BROKER" + AdvancedSubscriptions Feature = "ADVANCED_SUBSCRIPTIONS" + WebhookTransformations Feature = "WEBHOOK_TRANSFORMATIONS" + HADeployment Feature = "HA_DEPLOYMENT" + WebhookAnalytics Feature = "WEBHOOK_ANALYTICS" + MutualTLS Feature = "MUTUAL_TLS" + AsynqMonitoring Feature = "ASYNQ_MONITORING" + SynchronousWebhooks Feature = "SYNCHRONOUS_WEBHOOKS" + PortalLinks Feature = "PORTAL_LINKS" +) + +const ( + CommunityPlan PlanType = "community" + BusinessPlan PlanType = "business" + EnterprisePlan PlanType = "enterprise" +) + +// Properties will hold characteristics for features like organisation +// number limit, but it can also be empty, because certain feature don't need them +type Properties struct { + Limit int64 `mapstructure:"limit" json:"-"` + Allowed bool `json:"allowed"` +} + +type LicenseMetadata struct { + UserLimit int64 `mapstructure:"userLimit"` + OrgLimit int64 `mapstructure:"orgLimit"` + ProjectLimit int64 `mapstructure:"projectLimit"` +} diff --git a/internal/pkg/license/keygen/keygen.go b/internal/pkg/license/keygen/keygen.go new file mode 100644 index 0000000000..62bdd07b77 --- /dev/null +++ b/internal/pkg/license/keygen/keygen.go @@ -0,0 +1,456 @@ +package keygen + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "sync" + "time" + + "github.com/mitchellh/mapstructure" + + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/pkg/log" + "github.com/frain-dev/convoy/util" + + "github.com/google/uuid" + "github.com/keygen-sh/keygen-go/v3" +) + +type Licenser struct { + licenseKey string + license *keygen.License + planType PlanType + machineFingerprint string + featureList map[Feature]*Properties + + orgRepo datastore.OrganisationRepository + userRepo datastore.UserRepository + projectRepo datastore.ProjectRepository + + // only for community licenser + mu sync.RWMutex + enabledProjects map[string]bool +} + +type Config struct { + LicenseKey string + OrgRepo datastore.OrganisationRepository + ProjectRepo datastore.ProjectRepository + UserRepo datastore.UserRepository +} + +func init() { + keygen.Account = "8200bc0f-f64f-4a38-a9be-d2b16c8f0deb" + keygen.Product = "08d95b4d-4301-42f9-95af-9713e1b41a3a" + keygen.PublicKey = "14549f18dd23e4644aae6b6fd787e4df5f018bce0c7ae2edd29df83309ea76c2" +} + +func NewKeygenLicenser(c *Config) (*Licenser, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + if util.IsStringEmpty(c.LicenseKey) { + // no license key provided, allow access to only community features + return communityLicenser(ctx, c.OrgRepo, c.UserRepo, c.ProjectRepo) + } + + keygen.LicenseKey = c.LicenseKey + fingerprint := uuid.New().String() + + l, err := keygen.Validate(ctx, fingerprint) + if err != nil && !allowKeygenError(err) { + return nil, fmt.Errorf("failed to validate error: %v", err) + } + + err = checkExpiry(l) + if err != nil { + return nil, err + } + + if l.Metadata == nil { + return nil, fmt.Errorf("license has no metadata") + } + + featureList, err := getFeatureList(ctx, l) + if err != nil { + return nil, err + } + + p := l.Metadata["planType"] + if p == nil { + return nil, fmt.Errorf("license plan type unspecified in metadata") + } + + pt, ok := p.(string) + if !ok { + return nil, fmt.Errorf("license plan type is not a string") + } + + return &Licenser{ + machineFingerprint: fingerprint, + licenseKey: c.LicenseKey, + license: l, + orgRepo: c.OrgRepo, + userRepo: c.UserRepo, + projectRepo: c.ProjectRepo, + planType: PlanType(pt), + featureList: featureList, + }, err +} + +func (k *Licenser) ProjectEnabled(projectID string) bool { + k.mu.RLock() + defer k.mu.RUnlock() + if k.enabledProjects == nil { // not community licenser + return true + } + + return k.enabledProjects[projectID] +} + +func (k *Licenser) AddEnabledProject(projectID string) { + k.mu.Lock() + defer k.mu.Unlock() + if k.enabledProjects == nil { // not community licenser + return + } + + if len(k.enabledProjects) == projectLimit { + return + } + + k.enabledProjects[projectID] = true +} + +func (k *Licenser) RemoveEnabledProject(projectID string) { + k.mu.Lock() + defer k.mu.Unlock() + if k.enabledProjects == nil { // not community licenser + return + } + + delete(k.enabledProjects, projectID) +} + +func (k *Licenser) Activate() error { + if util.IsStringEmpty(k.licenseKey) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + machine, err := k.license.Activate(ctx, k.machineFingerprint) + if err != nil { + return fmt.Errorf("failed to activate machine") + } + + // Start a heartbeat monitor for the current machine + err = machine.Monitor(ctx) + if err != nil { + return fmt.Errorf("failed to start machine monitor") + } + + go func() { + // Listen for interrupt and deactivate the machine, if the instance crashes unexpectedly the + // heartbeat monitor helps to tell keygen that this machine should be deactivated + // See the Check-out/check-in licenses section on + // https://keygen.sh/docs/choosing-a-licensing-model/floating-licenses/ + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + <-quit + + if err := machine.Deactivate(ctx); err != nil { + log.WithError(err).Error("failed to deactivate machine") + } + }() + + return nil +} + +func allowKeygenError(err error) bool { + switch { + case errors.Is(err, keygen.ErrLicenseNotActivated): + return true + case errors.Is(err, keygen.ErrLicenseExpired): + return true + case errors.Is(err, keygen.ErrHeartbeatRequired): + return true + } + + return false +} + +var ErrLicenseExpired = errors.New("license expired") + +func checkExpiry(l *keygen.License) error { + if l.Expiry == nil { + return nil + } + + now := time.Now() + + if now.After(*l.Expiry) { + v := now.Sub(*l.Expiry) + + const days = 21 * 24 * time.Hour // 21 days + + if v < days { // expired in less than 21 days, allow instance to boot + daysAgo := int64(v.Hours() / 24) + log.Warnf("license expired %d days ago, access to features will be revoked in %d days", daysAgo, 21-daysAgo) + return nil + } + + return ErrLicenseExpired + } + + return nil +} + +func getFeatureList(ctx context.Context, l *keygen.License) (map[Feature]*Properties, error) { + entitlements, err := l.Entitlements(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load license entitlements: %v", err) + } + + if len(entitlements) == 0 { + return nil, fmt.Errorf("license has no entitlements") + } + + featureList := map[Feature]*Properties{} + for _, entitlement := range entitlements { + featureList[Feature(entitlement.Code)] = &Properties{Allowed: true} + } + + meta := LicenseMetadata{} + if l.Metadata != nil { + err = mapstructure.Decode(l.Metadata, &meta) + if err != nil { + return nil, fmt.Errorf("failed to decode license metadata: %v", err) + } + } + + if meta.OrgLimit != 0 { + featureList[CreateOrg] = &Properties{Limit: meta.OrgLimit} + } + + if meta.UserLimit != 0 { + featureList[CreateUser] = &Properties{Limit: meta.UserLimit} + } + + if meta.ProjectLimit != 0 { + featureList[CreateProject] = &Properties{Limit: meta.ProjectLimit} + } + + return featureList, err +} + +func (k *Licenser) CreateOrg(ctx context.Context) (bool, error) { + err := checkExpiry(k.license) + if err != nil { + return false, err + } + + c, err := k.orgRepo.CountOrganisations(ctx) + if err != nil { + return false, err + } + + p := k.featureList[CreateOrg] + + if p.Limit == -1 { // no limit + return true, nil + } + + if c >= p.Limit { + return false, nil + } + + return true, nil +} + +func (k *Licenser) CreateUser(ctx context.Context) (bool, error) { + err := checkExpiry(k.license) + if err != nil { + return false, err + } + + c, err := k.userRepo.CountUsers(ctx) + if err != nil { + return false, err + } + + p := k.featureList[CreateUser] + + if p.Limit == -1 { // no limit + return true, nil + } + + if c >= p.Limit { + return false, nil + } + + return true, nil +} + +func (k *Licenser) CreateProject(ctx context.Context) (bool, error) { + err := checkExpiry(k.license) + if err != nil { + return false, err + } + + c, err := k.projectRepo.CountProjects(ctx) + if err != nil { + return false, err + } + + p := k.featureList[CreateProject] + + if p.Limit == -1 { // no limit + return true, nil + } + + if c >= p.Limit { + return false, nil + } + + return true, nil +} + +func (k *Licenser) UseForwardProxy() bool { + if checkExpiry(k.license) != nil { + return false + } + + _, ok := k.featureList[UseForwardProxy] + return ok +} + +func (k *Licenser) CanExportPrometheusMetrics() bool { + if checkExpiry(k.license) != nil { + return false + } + + _, ok := k.featureList[ExportPrometheusMetrics] + return ok +} + +func (k *Licenser) AdvancedEndpointMgmt() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedEndpointMgmt] + return ok +} + +func (k *Licenser) AdvancedRetentionPolicy() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedWebhookArchiving] + return ok +} + +func (k *Licenser) AdvancedMsgBroker() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedMsgBroker] + return ok +} + +func (k *Licenser) AdvancedSubscriptions() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedSubscriptions] + return ok +} + +func (k *Licenser) Transformations() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[WebhookTransformations] + return ok +} + +func (k *Licenser) HADeployment() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[HADeployment] + return ok +} + +func (k *Licenser) WebhookAnalytics() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[WebhookAnalytics] + return ok +} + +func (k *Licenser) MutualTLS() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[MutualTLS] + return ok +} + +func (k *Licenser) AsynqMonitoring() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AsynqMonitoring] + return ok +} + +func (k *Licenser) SynchronousWebhooks() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[SynchronousWebhooks] + return ok +} + +func (k *Licenser) PortalLinks() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[PortalLinks] + return ok +} + +func (k *Licenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { + // only these guys have dynamic limits for now + for f := range k.featureList { + switch f { + case CreateOrg: + ok, err := k.CreateOrg(ctx) + if err != nil { + return nil, err + } + k.featureList[f].Allowed = ok + case CreateUser: + ok, err := k.CreateUser(ctx) + if err != nil { + return nil, err + } + k.featureList[f].Allowed = ok + case CreateProject: + ok, err := k.CreateProject(ctx) + if err != nil { + return nil, err + } + k.featureList[f].Allowed = ok + } + } + + return json.Marshal(k.featureList) +} diff --git a/internal/pkg/license/keygen/keygen_test.go b/internal/pkg/license/keygen/keygen_test.go new file mode 100644 index 0000000000..9d3f8861b6 --- /dev/null +++ b/internal/pkg/license/keygen/keygen_test.go @@ -0,0 +1,658 @@ +package keygen + +import ( + "context" + "encoding/json" + "errors" + "math" + "testing" + "time" + + "github.com/frain-dev/convoy/mocks" + "github.com/keygen-sh/keygen-go/v3" + "go.uber.org/mock/gomock" + + "github.com/stretchr/testify/require" +) + +func TestKeygenLicenserBoolMethods(t *testing.T) { + k := Licenser{featureList: map[Feature]*Properties{UseForwardProxy: {}}, license: &keygen.License{}} + require.True(t, k.UseForwardProxy()) + + k = Licenser{featureList: map[Feature]*Properties{UseForwardProxy: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.UseForwardProxy()) + + k = Licenser{featureList: map[Feature]*Properties{ExportPrometheusMetrics: {}}, license: &keygen.License{}} + require.True(t, k.CanExportPrometheusMetrics()) + + k = Licenser{featureList: map[Feature]*Properties{ExportPrometheusMetrics: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.CanExportPrometheusMetrics()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedEndpointMgmt: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedEndpointMgmt()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedEndpointMgmt: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedEndpointMgmt()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookArchiving: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedRetentionPolicy()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookArchiving: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedRetentionPolicy()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedMsgBroker: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedMsgBroker()) + k = Licenser{featureList: map[Feature]*Properties{AdvancedMsgBroker: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedMsgBroker()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedSubscriptions: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedSubscriptions()) + k = Licenser{featureList: map[Feature]*Properties{AdvancedSubscriptions: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedSubscriptions()) + + k = Licenser{featureList: map[Feature]*Properties{WebhookTransformations: {}}, license: &keygen.License{}} + require.True(t, k.Transformations()) + k = Licenser{featureList: map[Feature]*Properties{WebhookTransformations: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.Transformations()) + + k = Licenser{featureList: map[Feature]*Properties{HADeployment: {}}, license: &keygen.License{}} + require.True(t, k.HADeployment()) + k = Licenser{featureList: map[Feature]*Properties{HADeployment: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.HADeployment()) + + k = Licenser{featureList: map[Feature]*Properties{WebhookAnalytics: {}}, license: &keygen.License{}} + require.True(t, k.WebhookAnalytics()) + k = Licenser{featureList: map[Feature]*Properties{WebhookAnalytics: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.WebhookAnalytics()) + + k = Licenser{featureList: map[Feature]*Properties{MutualTLS: {}}, license: &keygen.License{}} + require.True(t, k.MutualTLS()) + k = Licenser{featureList: map[Feature]*Properties{MutualTLS: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.MutualTLS()) + + k = Licenser{featureList: map[Feature]*Properties{AsynqMonitoring: {}}, license: &keygen.License{}} + require.True(t, k.AsynqMonitoring()) + + k = Licenser{featureList: map[Feature]*Properties{AsynqMonitoring: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AsynqMonitoring()) + + k = Licenser{featureList: map[Feature]*Properties{SynchronousWebhooks: {}}, license: &keygen.License{}} + require.True(t, k.SynchronousWebhooks()) + k = Licenser{featureList: map[Feature]*Properties{SynchronousWebhooks: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.SynchronousWebhooks()) + + k = Licenser{featureList: map[Feature]*Properties{PortalLinks: {}}, license: &keygen.License{}} + require.True(t, k.PortalLinks()) + k = Licenser{featureList: map[Feature]*Properties{PortalLinks: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.PortalLinks()) + + k = Licenser{enabledProjects: map[string]bool{ + "12345": true, + }} + require.True(t, k.ProjectEnabled("12345")) + require.False(t, k.ProjectEnabled("5555")) + + // when k.enabledProjects is nil, should not add anything + k = Licenser{} + k.AddEnabledProject("11111") + require.False(t, k.enabledProjects["11111"]) + + k = Licenser{enabledProjects: map[string]bool{}} + k.AddEnabledProject("11111") + require.True(t, k.enabledProjects["11111"]) + + k = Licenser{enabledProjects: map[string]bool{"11111": true, "2222": true}} + k.RemoveEnabledProject("11111") + require.NotContains(t, k.enabledProjects, "11111") + require.Contains(t, k.enabledProjects, "2222") + + falseLicenser := Licenser{featureList: map[Feature]*Properties{}, license: &keygen.License{Expiry: timePtr(time.Now().Add(400000 * time.Hour))}} + + require.False(t, falseLicenser.UseForwardProxy()) + require.False(t, falseLicenser.PortalLinks()) + require.False(t, falseLicenser.CanExportPrometheusMetrics()) + require.False(t, falseLicenser.AdvancedEndpointMgmt()) + require.False(t, falseLicenser.AdvancedRetentionPolicy()) + require.False(t, falseLicenser.AdvancedMsgBroker()) + require.False(t, falseLicenser.AdvancedSubscriptions()) + require.False(t, falseLicenser.Transformations()) + require.False(t, falseLicenser.HADeployment()) + require.False(t, falseLicenser.WebhookAnalytics()) + require.False(t, falseLicenser.MutualTLS()) + require.False(t, falseLicenser.AsynqMonitoring()) + require.False(t, falseLicenser.SynchronousWebhooks()) +} + +func provideLicenser(ctrl *gomock.Controller, license *keygen.License, fl map[Feature]*Properties) *Licenser { + return &Licenser{ + featureList: fl, + license: license, + orgRepo: mocks.NewMockOrganisationRepository(ctrl), + userRepo: mocks.NewMockUserRepository(ctrl), + projectRepo: mocks.NewMockProjectRepository(ctrl), + } +} + +func TestKeygenLicenser_CreateProject(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + ctx context.Context + dbFn func(k *Licenser) + license *keygen.License + want bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_return_true", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_return_false_for_license_expired", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(-time.Hour * 40000))}, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + { + name: "should_return_false_for_limit_reached", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(1), nil) + }, + ctx: context.Background(), + want: false, + wantErr: false, + }, + { + name: "should_return_true_for_no_limit", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: -1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(math.MaxInt64), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_error_for_failed_to_count_org", + featureList: map[Feature]*Properties{ + CreateProject: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), errors.New("failed")) + }, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: "failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, tt.license, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.CreateProject(tt.ctx) + require.Equal(t, tt.want, got) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + }) + } +} + +func TestKeygenLicenser_CanCreateOrg(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + ctx context.Context + dbFn func(k *Licenser) + license *keygen.License + want bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_return_true", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_return_false_for_license_expired", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * -40000))}, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + { + name: "should_return_false_for_limit_reached", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(1), nil) + }, + ctx: context.Background(), + want: false, + wantErr: false, + }, + { + name: "should_return_true_for_no_limit", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: -1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(math.MaxInt64), nil) + }, + ctx: context.Background(), + want: true, + wantErr: false, + }, + { + name: "should_error_for_failed_to_count_org", + featureList: map[Feature]*Properties{ + CreateOrg: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), errors.New("failed")) + }, + ctx: context.Background(), + want: false, + wantErr: true, + wantErrMsg: "failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, tt.license, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.CreateOrg(tt.ctx) + require.Equal(t, tt.want, got) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + }) + } +} + +func TestKeygenLicenser_CanCreateUser(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + ctx context.Context + license *keygen.License + dbFn func(k *Licenser) + canCreateMember bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_return_true", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + canCreateMember: true, + wantErr: false, + }, + { + name: "should_return_false_for_limit_reached", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(1), nil) + }, + ctx: context.Background(), + canCreateMember: false, + wantErr: false, + }, + { + name: "should_return_true_for_no_limit", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: -1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + }, + ctx: context.Background(), + canCreateMember: true, + wantErr: false, + }, + { + name: "should_error_for_failed_to_count_org_members", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * 40000))}, + dbFn: func(k *Licenser) { + userRepository := k.userRepo.(*mocks.MockUserRepository) + userRepository.EXPECT().CountUsers(gomock.Any()).Return(int64(0), errors.New("failed")) + }, + ctx: context.Background(), + canCreateMember: false, + wantErr: true, + wantErrMsg: "failed", + }, + { + name: "should_error_for_license_expired", + featureList: map[Feature]*Properties{ + CreateUser: { + Limit: 1, + }, + }, + license: &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour * -40000))}, + ctx: context.Background(), + canCreateMember: false, + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, tt.license, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.CreateUser(tt.ctx) + require.Equal(t, tt.canCreateMember, got) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + }) + } +} + +func TestLicenser_FeatureListJSON(t *testing.T) { + tests := []struct { + name string + featureList map[Feature]*Properties + dbFn func(k *Licenser) + want json.RawMessage + wantErr bool + wantErrMsg string + }{ + { + name: "should_get_feature_list", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":true},"CREATE_PROJECT":{"allowed":true},"CREATE_USER":{"allowed":true}}`), + wantErr: false, + wantErrMsg: "", + }, + + { + name: "should_be_false_create_org", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(1), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":false},"CREATE_PROJECT":{"allowed":true},"CREATE_USER":{"allowed":true}}`), + wantErr: false, + wantErrMsg: "", + }, + + { + name: "should_be_false_create_user", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(1), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(0), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":true},"CREATE_PROJECT":{"allowed":true},"CREATE_USER":{"allowed":false}}`), + wantErr: false, + wantErrMsg: "", + }, + + { + name: "should_be_false_create_project", + featureList: map[Feature]*Properties{ + CreateOrg: {Limit: 1}, + CreateProject: {Limit: 1}, + CreateUser: {Limit: 1}, + AdvancedSubscriptions: {Limit: 1, Allowed: true}, + AdvancedEndpointMgmt: {Limit: 1, Allowed: true}, + }, + dbFn: func(k *Licenser) { + orgRepo := k.orgRepo.(*mocks.MockOrganisationRepository) + orgRepo.EXPECT().CountOrganisations(gomock.Any()).Return(int64(0), nil) + + userRepo := k.userRepo.(*mocks.MockUserRepository) + userRepo.EXPECT().CountUsers(gomock.Any()).Return(int64(0), nil) + + projectRepo := k.projectRepo.(*mocks.MockProjectRepository) + projectRepo.EXPECT().CountProjects(gomock.Any()).Return(int64(2), nil) + }, + want: []byte(`{"ADVANCED_ENDPOINT_MANAGEMENT":{"allowed":true},"ADVANCED_SUBSCRIPTIONS":{"allowed":true},"CREATE_ORG":{"allowed":true},"CREATE_PROJECT":{"allowed":false},"CREATE_USER":{"allowed":true}}`), + wantErr: false, + wantErrMsg: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + k := provideLicenser(ctrl, &keygen.License{Expiry: timePtr(time.Now().Add(time.Hour))}, tt.featureList) + + if tt.dbFn != nil { + tt.dbFn(k) + } + + got, err := k.FeatureListJSON(context.Background()) + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestCheckExpiry(t *testing.T) { + tests := []struct { + name string + expiry *time.Time + wantErr bool + wantErrMsg string + }{ + { + name: "No Expiry Date", + expiry: nil, + wantErr: false, + wantErrMsg: "", + }, + { + name: "License Expired within 21 Days", + expiry: timePtr(time.Now().Add(-10 * 24 * time.Hour)), // 10 days ago + wantErr: false, + }, + { + name: "License Expired beyond 21 Days", + expiry: timePtr(time.Now().Add(-22 * 24 * time.Hour)), // 22 days ago + wantErr: true, + wantErrMsg: ErrLicenseExpired.Error(), + }, + { + name: "License Not Yet Expired", + expiry: timePtr(time.Now().Add(5 * 24 * time.Hour)), // 5 days in the future + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + license := &keygen.License{ + Expiry: tt.expiry, + } + + err := checkExpiry(license) + + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + require.NoError(t, err) + }) + } +} + +// timePtr is a helper function to get a pointer to a time.Time value +func timePtr(t time.Time) *time.Time { + return &t +} diff --git a/internal/pkg/license/license.go b/internal/pkg/license/license.go new file mode 100644 index 0000000000..ed2ae65969 --- /dev/null +++ b/internal/pkg/license/license.go @@ -0,0 +1,45 @@ +package license + +import ( + "context" + "encoding/json" + + "github.com/frain-dev/convoy/internal/pkg/license/keygen" +) + +// Licenser interface provides methods to determine whether the specified license can utilise certain features in convoy. +type Licenser interface { + CreateOrg(ctx context.Context) (bool, error) + CreateUser(ctx context.Context) (bool, error) + CreateProject(ctx context.Context) (bool, error) + UseForwardProxy() bool + CanExportPrometheusMetrics() bool + AdvancedEndpointMgmt() bool + AdvancedSubscriptions() bool + Transformations() bool + AsynqMonitoring() bool + PortalLinks() bool + + // need more fleshing out + AdvancedRetentionPolicy() bool + AdvancedMsgBroker() bool + WebhookAnalytics() bool + HADeployment() bool + MutualTLS() bool + SynchronousWebhooks() bool + FeatureListJSON(ctx context.Context) (json.RawMessage, error) + + RemoveEnabledProject(projectID string) + AddEnabledProject(projectID string) + ProjectEnabled(projectID string) bool +} + +var _ Licenser = &keygen.Licenser{} + +type Config struct { + KeyGen keygen.Config +} + +func NewLicenser(c *Config) (Licenser, error) { + return keygen.NewKeygenLicenser(&c.KeyGen) +} diff --git a/internal/pkg/license/noop/noop.go b/internal/pkg/license/noop/noop.go new file mode 100644 index 0000000000..c1fcf897e4 --- /dev/null +++ b/internal/pkg/license/noop/noop.go @@ -0,0 +1,89 @@ +//go:build integration + +package noop + +import ( + "context" + "encoding/json" +) + +// Noop License is for testing only + +type Licenser struct{} + +func (Licenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { + return []byte{}, nil +} + +func NewLicenser() *Licenser { + return &Licenser{} +} + +func (Licenser) CreateOrg(ctx context.Context) (bool, error) { + return true, nil +} + +func (Licenser) CreateUser(ctx context.Context) (bool, error) { + return true, nil +} + +func (Licenser) CreateProject(ctx context.Context) (bool, error) { + return true, nil +} + +func (Licenser) UseForwardProxy() bool { + return true +} + +func (Licenser) CanExportPrometheusMetrics() bool { + return true +} + +func (Licenser) AdvancedEndpointMgmt() bool { + return true +} + +func (Licenser) AdvancedSubscriptions() bool { + return true +} + +func (Licenser) Transformations() bool { + return true +} + +func (Licenser) AsynqMonitoring() bool { + return true +} + +func (Licenser) AdvancedRetentionPolicy() bool { + return true +} + +func (Licenser) AdvancedMsgBroker() bool { + return true +} + +func (Licenser) WebhookAnalytics() bool { + return true +} + +func (Licenser) HADeployment() bool { + return true +} + +func (Licenser) MutualTLS() bool { + return true +} + +func (Licenser) SynchronousWebhooks() bool { + return true +} +func (Licenser) RemoveEnabledProject(_ string) {} +func (Licenser) ProjectEnabled(_ string) bool { + return true +} +func (Licenser) AddEnabledProject(_ string) {} + +func (Licenser) PortalLinks() bool { + return true +} diff --git a/internal/pkg/metrics/data_plane.go b/internal/pkg/metrics/data_plane.go index 60262c12fb..425ea0c08b 100644 --- a/internal/pkg/metrics/data_plane.go +++ b/internal/pkg/metrics/data_plane.go @@ -1,14 +1,18 @@ package metrics import ( + "sync" + "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/prometheus/client_golang/prometheus" - "sync" ) -var m *Metrics -var once sync.Once +var ( + m *Metrics + once sync.Once +) const ( projectLabel = "project" @@ -24,15 +28,15 @@ type Metrics struct { EventDeliveryLatency *prometheus.HistogramVec } -func GetDPInstance() *Metrics { +func GetDPInstance(licenser license.Licenser) *Metrics { once.Do(func() { - m = newMetrics(Reg()) + m = newMetrics(Reg(), licenser) }) return m } -func newMetrics(pr prometheus.Registerer) *Metrics { - m := InitMetrics() +func newMetrics(pr prometheus.Registerer, licenser license.Licenser) *Metrics { + m := InitMetrics(licenser) if m.IsEnabled && m.IngestTotal != nil && m.IngestConsumedTotal != nil && m.IngestErrorsTotal != nil { pr.MustRegister( @@ -45,15 +49,14 @@ func newMetrics(pr prometheus.Registerer) *Metrics { return m } -func InitMetrics() *Metrics { - +func InitMetrics(licenser license.Licenser) *Metrics { cfg, err := config.Get() if err != nil { return &Metrics{ IsEnabled: false, } } - if !cfg.Metrics.IsEnabled { + if !cfg.Metrics.IsEnabled || !licenser.CanExportPrometheusMetrics() { return &Metrics{ IsEnabled: false, } diff --git a/internal/pkg/middleware/middleware.go b/internal/pkg/middleware/middleware.go index e3125ff01a..b2dd322a67 100644 --- a/internal/pkg/middleware/middleware.go +++ b/internal/pkg/middleware/middleware.go @@ -6,14 +6,15 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/frain-dev/convoy" - "github.com/frain-dev/convoy/internal/pkg/limiter" - rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" "net/http" "strconv" "strings" "time" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/internal/pkg/limiter" + rlimiter "github.com/frain-dev/convoy/internal/pkg/limiter/redis" + "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/riandyrn/otelchi" diff --git a/internal/pkg/pubsub/amqp/client.go b/internal/pkg/pubsub/amqp/client.go index 40e40f5ac2..a7268decd1 100644 --- a/internal/pkg/pubsub/amqp/client.go +++ b/internal/pkg/pubsub/amqp/client.go @@ -3,7 +3,9 @@ package rqm import ( "context" "fmt" + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/log" @@ -23,10 +25,10 @@ type Amqp struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter) *Amqp { - +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser) *Amqp { return &Amqp{ Cfg: source.PubSub.Amqp, source: source, @@ -34,6 +36,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, } } @@ -79,7 +82,6 @@ func (k *Amqp) Verify() error { defer ch.Close() return nil - } func (k *Amqp) consume() { @@ -135,13 +137,12 @@ func (k *Amqp) consume() { false, // no-wait nil, // args ) - if err != nil { log.WithError(err).Error("failed to consume messages") return } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(k.licenser) mm.IncrementIngestTotal(k.source) for d := range messages { @@ -159,7 +160,6 @@ func (k *Amqp) consume() { mm.IncrementIngestConsumedTotal(k.source) } } else { - // Reject the message and send it to DLQ if err := d.Nack(false, false); err != nil { k.log.WithError(err).Error("failed to nack message") @@ -167,5 +167,4 @@ func (k *Amqp) consume() { } } } - } diff --git a/internal/pkg/pubsub/google/client.go b/internal/pkg/pubsub/google/client.go index 01ba5b5953..fffa79ef0b 100644 --- a/internal/pkg/pubsub/google/client.go +++ b/internal/pkg/pubsub/google/client.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/msgpack" @@ -24,9 +26,10 @@ type Google struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter) *Google { +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser) *Google { return &Google{ Cfg: source.PubSub.Google, source: source, @@ -34,6 +37,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, } } @@ -71,7 +75,6 @@ func (g *Google) Verify() error { func (g *Google) consume() { client, err := pubsub.NewClient(g.ctx, g.Cfg.ProjectID, option.WithCredentialsJSON(g.Cfg.ServiceAccount)) - if err != nil { g.log.WithError(err).Error("failed to create new pubsub client") } @@ -102,7 +105,7 @@ func (g *Google) consume() { attributes = emptyBytes } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(g.licenser) mm.IncrementIngestTotal(g.source) if err := g.handler(ctx, g.source, string(m.Data), attributes); err != nil { diff --git a/internal/pkg/pubsub/ingest.go b/internal/pkg/pubsub/ingest.go index 58b7caeea0..c0752325dc 100644 --- a/internal/pkg/pubsub/ingest.go +++ b/internal/pkg/pubsub/ingest.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/transform" @@ -39,9 +41,10 @@ type Ingest struct { table *memorystore.Table log log.StdLogger instanceId string + licenser license.Licenser } -func NewIngest(ctx context.Context, table *memorystore.Table, queue queue.Queuer, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) (*Ingest, error) { +func NewIngest(ctx context.Context, table *memorystore.Table, queue queue.Queuer, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) (*Ingest, error) { ctx = context.WithValue(ctx, ingestCtx, nil) i := &Ingest{ ctx: ctx, @@ -50,6 +53,7 @@ func NewIngest(ctx context.Context, table *memorystore.Table, queue queue.Queuer queue: queue, rateLimiter: rateLimiter, instanceId: instanceId, + licenser: licenser, sources: make(map[memorystore.Key]*PubSubSource), ticker: time.NewTicker(time.Duration(1) * time.Second), } @@ -118,12 +122,12 @@ func (i *Ingest) run() error { return errors.New("invalid source in memory store") } - ps, err := NewPubSubSource(i.ctx, &ss, i.handler, i.log, i.rateLimiter, i.instanceId) + ps, err := NewPubSubSource(i.ctx, &ss, i.handler, i.log, i.rateLimiter, i.licenser, i.instanceId) if err != nil { return err } - //ps.hash = key + // ps.hash = key ps.Start() i.sources[key] = ps } diff --git a/internal/pkg/pubsub/kafka/client.go b/internal/pkg/pubsub/kafka/client.go index 9660382905..2db2c8c16e 100644 --- a/internal/pkg/pubsub/kafka/client.go +++ b/internal/pkg/pubsub/kafka/client.go @@ -4,11 +4,13 @@ import ( "context" "crypto/tls" "fmt" + "time" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/msgpack" - "time" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/log" @@ -27,10 +29,11 @@ type Kafka struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser instanceId string } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) *Kafka { +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) *Kafka { return &Kafka{ Cfg: source.PubSub.Kafka, source: source, @@ -38,6 +41,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, instanceId: instanceId, } } @@ -107,7 +111,6 @@ func (k *Kafka) Verify() error { } return nil - } func (k *Kafka) consume() { @@ -160,7 +163,7 @@ func (k *Kafka) consume() { continue } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(k.licenser) mm.IncrementIngestTotal(k.source) var d D = m.Headers diff --git a/internal/pkg/pubsub/pubsub.go b/internal/pkg/pubsub/pubsub.go index 9ef88d4ea5..a4e4b35b10 100644 --- a/internal/pkg/pubsub/pubsub.go +++ b/internal/pkg/pubsub/pubsub.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "fmt" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -33,20 +35,22 @@ type PubSubSource struct { // The DB source source *datastore.Source + licenser license.Licenser + // This is a hash for the source config used to // track if an existing source config has been changed. hash string } -func NewPubSubSource(ctx context.Context, source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) (*PubSubSource, error) { - client, err := createClient(source, handler, log, rateLimiter, instanceId) +func NewPubSubSource(ctx context.Context, source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) (*PubSubSource, error) { + client, err := createClient(source, handler, log, rateLimiter, licenser, instanceId) if err != nil { return nil, err } ctx, cancelFunc := context.WithCancel(ctx) - pubSubSource := &PubSubSource{ctx: ctx, cancelFunc: cancelFunc, client: client, source: source} - //pubSubSource.hash = generateSourceKey(source) + pubSubSource := &PubSubSource{ctx: ctx, cancelFunc: cancelFunc, client: client, licenser: licenser, source: source} + // pubSubSource.hash = generateSourceKey(source) pubSubSource.cancelFunc = cancelFunc return pubSubSource, nil @@ -60,21 +64,21 @@ func (p *PubSubSource) Stop() { p.cancelFunc() } -func createClient(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) (PubSub, error) { +func createClient(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) (PubSub, error) { if source.PubSub.Type == datastore.SqsPubSub { - return sqs.New(source, handler, log, rateLimiter, instanceId), nil + return sqs.New(source, handler, log, rateLimiter, licenser, instanceId), nil } if source.PubSub.Type == datastore.GooglePubSub { - return google.New(source, handler, log, rateLimiter), nil + return google.New(source, handler, log, rateLimiter, licenser), nil } if source.PubSub.Type == datastore.KafkaPubSub { - return kafka.New(source, handler, log, rateLimiter, instanceId), nil + return kafka.New(source, handler, log, rateLimiter, licenser, instanceId), nil } if source.PubSub.Type == datastore.AmqpPubSub { - return rqm.New(source, handler, log, rateLimiter), nil + return rqm.New(source, handler, log, rateLimiter, licenser), nil } return nil, fmt.Errorf("pub sub type %s is not supported", source.PubSub.Type) diff --git a/internal/pkg/pubsub/sqs/client.go b/internal/pkg/pubsub/sqs/client.go index 624f7472a7..61740d79a6 100644 --- a/internal/pkg/pubsub/sqs/client.go +++ b/internal/pkg/pubsub/sqs/client.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" + "sync" + "time" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/internal/pkg/metrics" "github.com/frain-dev/convoy/pkg/msgpack" - "sync" - "time" "github.com/frain-dev/convoy/util" @@ -31,10 +33,11 @@ type Sqs struct { handler datastore.PubSubHandler log log.StdLogger rateLimiter limiter.RateLimiter + licenser license.Licenser instanceId string } -func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, instanceId string) *Sqs { +func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdLogger, rateLimiter limiter.RateLimiter, licenser license.Licenser, instanceId string) *Sqs { return &Sqs{ Cfg: source.PubSub.Sqs, source: source, @@ -42,6 +45,7 @@ func New(source *datastore.Source, handler datastore.PubSubHandler, log log.StdL handler: handler, log: log, rateLimiter: rateLimiter, + licenser: licenser, instanceId: instanceId, } } @@ -60,7 +64,6 @@ func (s *Sqs) Verify() error { Region: aws.String(s.Cfg.DefaultRegion), Credentials: credentials.NewStaticCredentials(s.Cfg.AccessKeyID, s.Cfg.SecretKey, ""), }) - if err != nil { log.WithError(err).Error("failed to create new session - sqs") return ErrInvalidCredentials @@ -95,7 +98,6 @@ func (s *Sqs) consume() { url, err := svc.GetQueueUrl(&sqs.GetQueueUrlInput{ QueueName: &s.Cfg.QueueName, }) - if err != nil { s.log.WithError(err).Error("failed to fetch queue url - sqs") } @@ -146,13 +148,12 @@ func (s *Sqs) consume() { WaitTimeSeconds: aws.Int64(1), MessageAttributeNames: []*string{&allAttr}, }) - if err != nil { s.log.WithError(err).Error("failed to fetch message - sqs") continue } - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(s.licenser) mm.IncrementIngestTotal(s.source) var wg sync.WaitGroup @@ -197,7 +198,6 @@ func (s *Sqs) consume() { s.log.WithError(err).Error("failed to delete message") } } - }(message) wg.Wait() diff --git a/mocks/license.go b/mocks/license.go new file mode 100644 index 0000000000..5ff9840ca8 --- /dev/null +++ b/mocks/license.go @@ -0,0 +1,321 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/license/license.go +// +// Generated by this command: +// +// mockgen --source internal/pkg/license/license.go --destination mocks/license.go -package mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + json "encoding/json" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockLicenser is a mock of Licenser interface. +type MockLicenser struct { + ctrl *gomock.Controller + recorder *MockLicenserMockRecorder +} + +// MockLicenserMockRecorder is the mock recorder for MockLicenser. +type MockLicenserMockRecorder struct { + mock *MockLicenser +} + +// NewMockLicenser creates a new mock instance. +func NewMockLicenser(ctrl *gomock.Controller) *MockLicenser { + mock := &MockLicenser{ctrl: ctrl} + mock.recorder = &MockLicenserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLicenser) EXPECT() *MockLicenserMockRecorder { + return m.recorder +} + +// AddEnabledProject mocks base method. +func (m *MockLicenser) AddEnabledProject(projectID string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddEnabledProject", projectID) +} + +// AddEnabledProject indicates an expected call of AddEnabledProject. +func (mr *MockLicenserMockRecorder) AddEnabledProject(projectID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddEnabledProject", reflect.TypeOf((*MockLicenser)(nil).AddEnabledProject), projectID) +} + +// AdvancedEndpointMgmt mocks base method. +func (m *MockLicenser) AdvancedEndpointMgmt() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedEndpointMgmt") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedEndpointMgmt indicates an expected call of AdvancedEndpointMgmt. +func (mr *MockLicenserMockRecorder) AdvancedEndpointMgmt() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedEndpointMgmt", reflect.TypeOf((*MockLicenser)(nil).AdvancedEndpointMgmt)) +} + +// AdvancedMsgBroker mocks base method. +func (m *MockLicenser) AdvancedMsgBroker() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedMsgBroker") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedMsgBroker indicates an expected call of AdvancedMsgBroker. +func (mr *MockLicenserMockRecorder) AdvancedMsgBroker() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedMsgBroker", reflect.TypeOf((*MockLicenser)(nil).AdvancedMsgBroker)) +} + +// AdvancedRetentionPolicy mocks base method. +func (m *MockLicenser) AdvancedRetentionPolicy() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedRetentionPolicy") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedRetentionPolicy indicates an expected call of AdvancedRetentionPolicy. +func (mr *MockLicenserMockRecorder) AdvancedRetentionPolicy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedRetentionPolicy", reflect.TypeOf((*MockLicenser)(nil).AdvancedRetentionPolicy)) +} + +// AdvancedSubscriptions mocks base method. +func (m *MockLicenser) AdvancedSubscriptions() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedSubscriptions") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedSubscriptions indicates an expected call of AdvancedSubscriptions. +func (mr *MockLicenserMockRecorder) AdvancedSubscriptions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedSubscriptions", reflect.TypeOf((*MockLicenser)(nil).AdvancedSubscriptions)) +} + +// AsynqMonitoring mocks base method. +func (m *MockLicenser) AsynqMonitoring() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AsynqMonitoring") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AsynqMonitoring indicates an expected call of AsynqMonitoring. +func (mr *MockLicenserMockRecorder) AsynqMonitoring() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsynqMonitoring", reflect.TypeOf((*MockLicenser)(nil).AsynqMonitoring)) +} + +// CanExportPrometheusMetrics mocks base method. +func (m *MockLicenser) CanExportPrometheusMetrics() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CanExportPrometheusMetrics") + ret0, _ := ret[0].(bool) + return ret0 +} + +// CanExportPrometheusMetrics indicates an expected call of CanExportPrometheusMetrics. +func (mr *MockLicenserMockRecorder) CanExportPrometheusMetrics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanExportPrometheusMetrics", reflect.TypeOf((*MockLicenser)(nil).CanExportPrometheusMetrics)) +} + +// CreateOrg mocks base method. +func (m *MockLicenser) CreateOrg(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrg", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateOrg indicates an expected call of CreateOrg. +func (mr *MockLicenserMockRecorder) CreateOrg(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrg", reflect.TypeOf((*MockLicenser)(nil).CreateOrg), ctx) +} + +// CreateProject mocks base method. +func (m *MockLicenser) CreateProject(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateProject", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateProject indicates an expected call of CreateProject. +func (mr *MockLicenserMockRecorder) CreateProject(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProject", reflect.TypeOf((*MockLicenser)(nil).CreateProject), ctx) +} + +// CreateUser mocks base method. +func (m *MockLicenser) CreateUser(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateUser", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateUser indicates an expected call of CreateUser. +func (mr *MockLicenserMockRecorder) CreateUser(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateUser", reflect.TypeOf((*MockLicenser)(nil).CreateUser), ctx) +} + +// FeatureListJSON mocks base method. +func (m *MockLicenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FeatureListJSON", ctx) + ret0, _ := ret[0].(json.RawMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FeatureListJSON indicates an expected call of FeatureListJSON. +func (mr *MockLicenserMockRecorder) FeatureListJSON(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FeatureListJSON", reflect.TypeOf((*MockLicenser)(nil).FeatureListJSON), ctx) +} + +// HADeployment mocks base method. +func (m *MockLicenser) HADeployment() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HADeployment") + ret0, _ := ret[0].(bool) + return ret0 +} + +// HADeployment indicates an expected call of HADeployment. +func (mr *MockLicenserMockRecorder) HADeployment() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HADeployment", reflect.TypeOf((*MockLicenser)(nil).HADeployment)) +} + +// MutualTLS mocks base method. +func (m *MockLicenser) MutualTLS() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MutualTLS") + ret0, _ := ret[0].(bool) + return ret0 +} + +// MutualTLS indicates an expected call of MutualTLS. +func (mr *MockLicenserMockRecorder) MutualTLS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MutualTLS", reflect.TypeOf((*MockLicenser)(nil).MutualTLS)) +} + +// PortalLinks mocks base method. +func (m *MockLicenser) PortalLinks() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PortalLinks") + ret0, _ := ret[0].(bool) + return ret0 +} + +// PortalLinks indicates an expected call of PortalLinks. +func (mr *MockLicenserMockRecorder) PortalLinks() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PortalLinks", reflect.TypeOf((*MockLicenser)(nil).PortalLinks)) +} + +// ProjectEnabled mocks base method. +func (m *MockLicenser) ProjectEnabled(projectID string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProjectEnabled", projectID) + ret0, _ := ret[0].(bool) + return ret0 +} + +// ProjectEnabled indicates an expected call of ProjectEnabled. +func (mr *MockLicenserMockRecorder) ProjectEnabled(projectID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProjectEnabled", reflect.TypeOf((*MockLicenser)(nil).ProjectEnabled), projectID) +} + +// RemoveEnabledProject mocks base method. +func (m *MockLicenser) RemoveEnabledProject(projectID string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RemoveEnabledProject", projectID) +} + +// RemoveEnabledProject indicates an expected call of RemoveEnabledProject. +func (mr *MockLicenserMockRecorder) RemoveEnabledProject(projectID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveEnabledProject", reflect.TypeOf((*MockLicenser)(nil).RemoveEnabledProject), projectID) +} + +// SynchronousWebhooks mocks base method. +func (m *MockLicenser) SynchronousWebhooks() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SynchronousWebhooks") + ret0, _ := ret[0].(bool) + return ret0 +} + +// SynchronousWebhooks indicates an expected call of SynchronousWebhooks. +func (mr *MockLicenserMockRecorder) SynchronousWebhooks() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SynchronousWebhooks", reflect.TypeOf((*MockLicenser)(nil).SynchronousWebhooks)) +} + +// Transformations mocks base method. +func (m *MockLicenser) Transformations() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Transformations") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Transformations indicates an expected call of Transformations. +func (mr *MockLicenserMockRecorder) Transformations() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Transformations", reflect.TypeOf((*MockLicenser)(nil).Transformations)) +} + +// UseForwardProxy mocks base method. +func (m *MockLicenser) UseForwardProxy() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UseForwardProxy") + ret0, _ := ret[0].(bool) + return ret0 +} + +// UseForwardProxy indicates an expected call of UseForwardProxy. +func (mr *MockLicenserMockRecorder) UseForwardProxy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UseForwardProxy", reflect.TypeOf((*MockLicenser)(nil).UseForwardProxy)) +} + +// WebhookAnalytics mocks base method. +func (m *MockLicenser) WebhookAnalytics() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WebhookAnalytics") + ret0, _ := ret[0].(bool) + return ret0 +} + +// WebhookAnalytics indicates an expected call of WebhookAnalytics. +func (mr *MockLicenserMockRecorder) WebhookAnalytics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WebhookAnalytics", reflect.TypeOf((*MockLicenser)(nil).WebhookAnalytics)) +} diff --git a/mocks/repository.go b/mocks/repository.go index 807456ccdb..4d378aae97 100644 --- a/mocks/repository.go +++ b/mocks/repository.go @@ -657,6 +657,21 @@ func (m *MockProjectRepository) EXPECT() *MockProjectRepositoryMockRecorder { return m.recorder } +// CountProjects mocks base method. +func (m *MockProjectRepository) CountProjects(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountProjects", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountProjects indicates an expected call of CountProjects. +func (mr *MockProjectRepositoryMockRecorder) CountProjects(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountProjects", reflect.TypeOf((*MockProjectRepository)(nil).CountProjects), ctx) +} + // CreateProject mocks base method. func (m *MockProjectRepository) CreateProject(arg0 context.Context, arg1 *datastore.Project) error { m.ctrl.T.Helper() @@ -781,6 +796,21 @@ func (m *MockOrganisationRepository) EXPECT() *MockOrganisationRepositoryMockRec return m.recorder } +// CountOrganisations mocks base method. +func (m *MockOrganisationRepository) CountOrganisations(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountOrganisations", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountOrganisations indicates an expected call of CountOrganisations. +func (mr *MockOrganisationRepositoryMockRecorder) CountOrganisations(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountOrganisations", reflect.TypeOf((*MockOrganisationRepository)(nil).CountOrganisations), ctx) +} + // CreateOrganisation mocks base method. func (m *MockOrganisationRepository) CreateOrganisation(arg0 context.Context, arg1 *datastore.Organisation) error { m.ctrl.T.Helper() @@ -2040,6 +2070,21 @@ func (m *MockUserRepository) EXPECT() *MockUserRepositoryMockRecorder { return m.recorder } +// CountUsers mocks base method. +func (m *MockUserRepository) CountUsers(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountUsers", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountUsers indicates an expected call of CountUsers. +func (mr *MockUserRepositoryMockRecorder) CountUsers(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountUsers", reflect.TypeOf((*MockUserRepository)(nil).CountUsers), ctx) +} + // CreateUser mocks base method. func (m *MockUserRepository) CreateUser(arg0 context.Context, arg1 *datastore.User) error { m.ctrl.T.Helper() @@ -2114,22 +2159,6 @@ func (mr *MockUserRepositoryMockRecorder) FindUserByToken(arg0, arg1 any) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindUserByToken", reflect.TypeOf((*MockUserRepository)(nil).FindUserByToken), arg0, arg1) } -// LoadUsersPaged mocks base method. -func (m *MockUserRepository) LoadUsersPaged(arg0 context.Context, arg1 datastore.Pageable) ([]datastore.User, datastore.PaginationData, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadUsersPaged", arg0, arg1) - ret0, _ := ret[0].([]datastore.User) - ret1, _ := ret[1].(datastore.PaginationData) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// LoadUsersPaged indicates an expected call of LoadUsersPaged. -func (mr *MockUserRepositoryMockRecorder) LoadUsersPaged(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadUsersPaged", reflect.TypeOf((*MockUserRepository)(nil).LoadUsersPaged), arg0, arg1) -} - // UpdateUser mocks base method. func (m *MockUserRepository) UpdateUser(ctx context.Context, user *datastore.User) error { m.ctrl.T.Helper() diff --git a/net/dispatcher.go b/net/dispatcher.go index 6077daddde..cb66d7edb4 100644 --- a/net/dispatcher.go +++ b/net/dispatcher.go @@ -11,6 +11,8 @@ import ( "net/url" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/pkg/httpheader" "github.com/frain-dev/convoy/pkg/log" @@ -21,7 +23,7 @@ type Dispatcher struct { client *http.Client } -func NewDispatcher(httpProxy string, enforceSecure bool) (*Dispatcher, error) { +func NewDispatcher(httpProxy string, licenser license.Licenser, enforceSecure bool) (*Dispatcher, error) { d := &Dispatcher{client: &http.Client{}} tr := &http.Transport{ @@ -32,13 +34,15 @@ func NewDispatcher(httpProxy string, enforceSecure bool) (*Dispatcher, error) { ExpectContinueTimeout: 1 * time.Second, } - proxyUrl, isValid, err := d.setProxy(httpProxy) - if err != nil { - return nil, err - } + if licenser.UseForwardProxy() { + proxyUrl, isValid, err := d.setProxy(httpProxy) + if err != nil { + return nil, err + } - if isValid { - tr.Proxy = http.ProxyURL(proxyUrl) + if isValid { + tr.Proxy = http.ProxyURL(proxyUrl) + } } // if enforceSecure is false, allow self-signed certificates, susceptible to MITM attacks. diff --git a/net/dispatcher_test.go b/net/dispatcher_test.go index c309b2d8bf..1af4788e69 100644 --- a/net/dispatcher_test.go +++ b/net/dispatcher_test.go @@ -9,6 +9,11 @@ import ( "testing" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + + "github.com/frain-dev/convoy/mocks" + "go.uber.org/mock/gomock" + "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/pkg/httpheader" "github.com/jarcoal/httpmock" @@ -294,3 +299,70 @@ func TestDispatcher_SendRequest(t *testing.T) { }) } } + +func TestNewDispatcher(t *testing.T) { + type args struct { + httpProxy string + enforceSecure bool + } + tests := []struct { + name string + args args + mockFn func(licenser license.Licenser) + wantProxy bool + wantErr bool + wantErrMsg string + }{ + { + name: "should_set_proxy", + args: args{ + httpProxy: "https://21.3.32.33:443", + enforceSecure: false, + }, + mockFn: func(licenser license.Licenser) { + l := licenser.(*mocks.MockLicenser) + l.EXPECT().UseForwardProxy().Return(true) + }, + wantProxy: true, + wantErr: false, + wantErrMsg: "", + }, + { + name: "should_not_set_proxy", + args: args{ + httpProxy: "https://21.3.32.33:443", + enforceSecure: false, + }, + mockFn: func(licenser license.Licenser) { + l := licenser.(*mocks.MockLicenser) + l.EXPECT().UseForwardProxy().Return(false) + }, + wantProxy: false, + wantErr: false, + wantErrMsg: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + licenser := mocks.NewMockLicenser(ctrl) + if tt.mockFn != nil { + tt.mockFn(licenser) + } + d, err := NewDispatcher(tt.args.httpProxy, licenser, tt.args.enforceSecure) + if tt.wantErr { + require.Error(t, err) + require.Equal(t, tt.wantErrMsg, err.Error()) + return + } + + require.NoError(t, err) + + if tt.wantProxy { + require.NotNil(t, d.client.Transport.(*http.Transport).Proxy) + } + }) + } +} diff --git a/services/create_endpoint.go b/services/create_endpoint.go index 2a52656d67..ea71ef746a 100644 --- a/services/create_endpoint.go +++ b/services/create_endpoint.go @@ -6,6 +6,10 @@ import ( "net/http" "time" + "github.com/frain-dev/convoy" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/cache" "github.com/frain-dev/convoy/datastore" @@ -19,6 +23,7 @@ type CreateEndpointService struct { PortalLinkRepo datastore.PortalLinkRepository EndpointRepo datastore.EndpointRepository ProjectRepo datastore.ProjectRepository + Licenser license.Licenser E models.CreateEndpoint ProjectID string @@ -68,6 +73,11 @@ func (a *CreateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e UpdatedAt: time.Now(), } + if !a.Licenser.AdvancedEndpointMgmt() { + // switch to default timeout + endpoint.HttpTimeout = convoy.HTTP_TIMEOUT + } + if util.IsStringEmpty(endpoint.AppID) { endpoint.AppID = endpoint.UID } diff --git a/services/create_endpoint_test.go b/services/create_endpoint_test.go index 0117ffd6e8..aced45986d 100644 --- a/services/create_endpoint_test.go +++ b/services/create_endpoint_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/mocks" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -18,6 +20,7 @@ func provideCreateEndpointService(ctrl *gomock.Controller, e models.CreateEndpoi Cache: mocks.NewMockCache(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), ProjectRepo: mocks.NewMockProjectRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), E: e, ProjectID: projectID, } @@ -50,6 +53,7 @@ func TestCreateEndpointService_Run(t *testing.T) { SupportEmail: "endpoint@test.com", IsDisabled: false, SlackWebhookURL: "https://google.com", + HttpTimeout: 30, Secret: "1234", URL: "https://google.com", Description: "test_endpoint", @@ -63,7 +67,13 @@ func TestCreateEndpointService_Run(t *testing.T) { Return(project, nil) a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) - a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil) + a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Cond(func(x any) bool { + endpoint := x.(*datastore.Endpoint) + return endpoint.HttpTimeout == 30 + }), gomock.Any()).Times(1).Return(nil) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantEndpoint: &datastore.Endpoint{ Name: "endpoint", @@ -74,6 +84,56 @@ func TestCreateEndpointService_Run(t *testing.T) { {Value: "1234"}, }, AdvancedSignatures: true, + HttpTimeout: 30, + Url: "https://google.com", + Description: "test_endpoint", + RateLimit: 0, + Status: datastore.ActiveEndpointStatus, + RateLimitDuration: 0, + }, + wantErr: false, + }, + { + name: "should_default_http_timeout_endpoint_for_license_check", + args: args{ + ctx: ctx, + e: models.CreateEndpoint{ + Name: "endpoint", + SupportEmail: "endpoint@test.com", + IsDisabled: false, + SlackWebhookURL: "https://google.com", + Secret: "1234", + URL: "https://google.com", + HttpTimeout: 3, + Description: "test_endpoint", + }, + g: project, + }, + dbFn: func(app *CreateEndpointService) { + p, _ := app.ProjectRepo.(*mocks.MockProjectRepository) + p.EXPECT().FetchProjectByID(gomock.Any(), gomock.Any()). + Times(1). + Return(project, nil) + + a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) + a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Cond(func(x any) bool { + endpoint := x.(*datastore.Endpoint) + return endpoint.HttpTimeout == convoy.HTTP_TIMEOUT + }), gomock.Any()).Times(1).Return(nil) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) + }, + wantEndpoint: &datastore.Endpoint{ + Name: "endpoint", + SupportEmail: "endpoint@test.com", + SlackWebhookURL: "https://google.com", + ProjectID: project.UID, + Secrets: []datastore.Secret{ + {Value: "1234"}, + }, + AdvancedSignatures: true, + HttpTimeout: convoy.HTTP_TIMEOUT, Url: "https://google.com", Description: "test_endpoint", RateLimit: 0, @@ -111,6 +171,9 @@ func TestCreateEndpointService_Run(t *testing.T) { a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantEndpoint: &datastore.Endpoint{ ProjectID: project.UID, @@ -156,6 +219,9 @@ func TestCreateEndpointService_Run(t *testing.T) { a, _ := app.EndpointRepo.(*mocks.MockEndpointRepository) a.EXPECT().CreateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(errors.New("failed")) + + licenser, _ := app.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantErr: true, wantErrMsg: "an error occurred while adding endpoint", diff --git a/services/create_organisation.go b/services/create_organisation.go index 33e595253a..3e4e7e8220 100644 --- a/services/create_organisation.go +++ b/services/create_organisation.go @@ -2,9 +2,12 @@ package services import ( "context" + "errors" "fmt" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/dchest/uniuri" "github.com/frain-dev/convoy/api/models" "github.com/frain-dev/convoy/auth" @@ -21,10 +24,22 @@ type CreateOrganisationService struct { OrgMemberRepo datastore.OrganisationMemberRepository NewOrg *models.Organisation User *datastore.User + Licenser license.Licenser } +var ErrOrgLimit = errors.New("your instance has reached it's organisation limit, upgrade to create new organisations") + func (co *CreateOrganisationService) Run(ctx context.Context) (*datastore.Organisation, error) { - err := util.Validate(co.NewOrg) + ok, err := co.Licenser.CreateOrg(ctx) + if err != nil { + return nil, &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return nil, &ServiceError{ErrMsg: ErrOrgLimit.Error(), Err: ErrOrgLimit} + } + + err = util.Validate(co.NewOrg) if err != nil { return nil, &ServiceError{ErrMsg: err.Error()} } diff --git a/services/create_organisation_test.go b/services/create_organisation_test.go index e80458291a..ee61979b5e 100644 --- a/services/create_organisation_test.go +++ b/services/create_organisation_test.go @@ -17,6 +17,7 @@ func provideCreateOrganisationService(ctrl *gomock.Controller, newOrg *models.Or return &CreateOrganisationService{ OrgRepo: mocks.NewMockOrganisationRepository(ctrl), OrgMemberRepo: mocks.NewMockOrganisationMemberRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), NewOrg: newOrg, User: user, } @@ -53,6 +54,9 @@ func TestCreateOrganisationService_Run(t *testing.T) { om, _ := os.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -63,6 +67,10 @@ func TestCreateOrganisationService_Run(t *testing.T) { newOrg: &models.Organisation{Name: ""}, user: &datastore.User{UID: "1234"}, }, + dbFn: func(os *CreateOrganisationService) { + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + }, wantErr: true, wantErrMsg: "organisation name is required", }, @@ -74,6 +82,9 @@ func TestCreateOrganisationService_Run(t *testing.T) { user: &datastore.User{UID: "1234"}, }, dbFn: func(os *CreateOrganisationService) { + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + a, _ := os.OrgRepo.(*mocks.MockOrganisationRepository) a.EXPECT().CreateOrganisation(gomock.Any(), gomock.Any()). Times(1).Return(errors.New("failed")) @@ -81,6 +92,20 @@ func TestCreateOrganisationService_Run(t *testing.T) { wantErr: true, wantErrMsg: "failed to create organisation", }, + { + name: "should_fail_to_create_organisation_for_license_check", + args: args{ + ctx: ctx, + newOrg: &models.Organisation{Name: "new_org"}, + user: &datastore.User{UID: "1234"}, + }, + dbFn: func(os *CreateOrganisationService) { + licenser, _ := os.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrMsg: ErrOrgLimit.Error(), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/services/create_subscription.go b/services/create_subscription.go index 1b6a30cd9d..76c749c04c 100644 --- a/services/create_subscription.go +++ b/services/create_subscription.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "errors" - "gopkg.in/guregu/null.v4" "net/http" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/oklog/ulid/v2" "github.com/frain-dev/convoy/api/models" @@ -27,6 +29,7 @@ type CreateSubscriptionService struct { SourceRepo datastore.SourceRepository Project *datastore.Project NewSubscription *models.CreateSubscription + Licenser license.Licenser } func (s *CreateSubscriptionService) Run(ctx context.Context) (*datastore.Subscription, error) { @@ -72,22 +75,28 @@ func (s *CreateSubscriptionService) Run(ctx context.Context) (*datastore.Subscri Type: datastore.SubscriptionTypeAPI, SourceID: s.NewSubscription.SourceID, EndpointID: s.NewSubscription.EndpointID, - Function: null.StringFrom(s.NewSubscription.Function), RetryConfig: retryConfig, AlertConfig: s.NewSubscription.AlertConfig.Transform(), - FilterConfig: s.NewSubscription.FilterConfig.Transform(), RateLimitConfig: s.NewSubscription.RateLimitConfig.Transform(), CreatedAt: time.Now(), UpdatedAt: time.Now(), } + if s.Licenser.AdvancedSubscriptions() { + subscription.FilterConfig = s.NewSubscription.FilterConfig.Transform() + } + + if s.Licenser.Transformations() { + subscription.Function = null.StringFrom(s.NewSubscription.Function) + } + if subscription.FilterConfig == nil { subscription.FilterConfig = &datastore.FilterConfiguration{} } - if subscription.FilterConfig.EventTypes == nil || len(subscription.FilterConfig.EventTypes) == 0 { + if len(subscription.FilterConfig.EventTypes) == 0 { subscription.FilterConfig.EventTypes = []string{"*"} } diff --git a/services/create_subscription_test.go b/services/create_subscription_test.go index d9b3884243..5525a8ec12 100644 --- a/services/create_subscription_test.go +++ b/services/create_subscription_test.go @@ -3,7 +3,11 @@ package services import ( "context" "errors" + "reflect" "testing" + "time" + + "gopkg.in/guregu/null.v4" "github.com/frain-dev/convoy/mocks" "github.com/stretchr/testify/require" @@ -18,6 +22,7 @@ func provideCreateSubscriptionService(ctrl *gomock.Controller, project *datastor SubRepo: mocks.NewMockSubscriptionRepository(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), SourceRepo: mocks.NewMockSourceRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), Project: project, NewSubscription: newSub, } @@ -58,6 +63,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { EndpointID: "endpoint-id-1", }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -78,6 +87,79 @@ func TestCreateSubscriptionService_Run(t *testing.T) { ) }, }, + { + name: "should skip filter config & function fields for subscription for outgoing project", + args: args{ + ctx: ctx, + newSubscription: &models.CreateSubscription{ + Name: "sub 1", + SourceID: "source-id-1", + EndpointID: "endpoint-id-1", + Function: "console.log", + FilterConfig: &models.FilterConfiguration{ + EventTypes: []string{"invoice.created"}, + Filter: models.FS{ + Headers: datastore.M{"x-msg-type": "stream-data"}, + Body: datastore.M{"offset": "1234"}, + }, + }, + }, + project: &datastore.Project{UID: "12345", Type: datastore.OutgoingProject, Config: &datastore.ProjectConfig{MultipleEndpointSubscriptions: false}}, + }, + wantSubscription: &datastore.Subscription{ + Name: "sub 1", + Type: datastore.SubscriptionTypeAPI, + SourceID: "source-id-1", + EndpointID: "endpoint-id-1", + }, + dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(false) + licenser.EXPECT().Transformations().Times(1).Return(false) + + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) + s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Cond(func(x any) bool { + sub := x.(*datastore.Subscription) + var uid string + uid, sub.UID = sub.UID, "" + sub.CreatedAt, sub.UpdatedAt = time.Time{}, time.Time{} + + c := &datastore.Subscription{ + Name: "sub 1", + SourceID: "source-id-1", + EndpointID: "endpoint-id-1", + ProjectID: "12345", + Function: null.String{}, + Type: datastore.SubscriptionTypeAPI, + FilterConfig: &datastore.FilterConfiguration{ + EventTypes: []string{"*"}, + Filter: datastore.FilterSchema{ + Headers: datastore.M{}, + Body: datastore.M{}, + }, + }, + } + + ok := reflect.DeepEqual(sub, c) + sub.UID = uid + return ok + })).Times(1).Return(nil) + + s.EXPECT().CountEndpointSubscriptions(gomock.Any(), "12345", "endpoint-id-1"). + Times(1). + Return(int64(0), nil) + + a, _ := ss.EndpointRepo.(*mocks.MockEndpointRepository) + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()). + Times(1).Return( + &datastore.Endpoint{ + UID: "endpoint-id-1", + ProjectID: "12345", + }, + nil, + ) + }, + }, { name: "should fail to count endpoint subscriptions for outgoing project if multi endpoints for subscriptions is false", args: args{ @@ -132,6 +214,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { EndpointID: "endpoint-id-1", }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -202,6 +288,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { EndpointID: "endpoint-id-1", }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -355,6 +445,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { }, }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). @@ -393,6 +487,10 @@ func TestCreateSubscriptionService_Run(t *testing.T) { }, }, dbFn: func(ss *CreateSubscriptionService) { + licenser, _ := ss.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + licenser.EXPECT().Transformations().Times(1).Return(true) + s, _ := ss.SubRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CreateSubscription(gomock.Any(), gomock.Any(), gomock.Any()). Times(1). diff --git a/services/invite_user.go b/services/invite_user.go index 08661ecdb7..a6c1660b37 100644 --- a/services/invite_user.go +++ b/services/invite_user.go @@ -3,10 +3,12 @@ package services import ( "context" "fmt" - "github.com/frain-dev/convoy/pkg/msgpack" "strings" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/pkg/msgpack" + "github.com/dchest/uniuri" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth" @@ -26,9 +28,19 @@ type InviteUserService struct { Role auth.Role User *datastore.User Organisation *datastore.Organisation + Licenser license.Licenser } func (iu *InviteUserService) Run(ctx context.Context) (*datastore.OrganisationInvite, error) { + ok, err := iu.Licenser.CreateUser(ctx) + if err != nil { + return nil, &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return nil, &ServiceError{ErrMsg: ErrUserLimit.Error()} + } + iv := &datastore.OrganisationInvite{ UID: ulid.Make().String(), OrganisationID: iu.Organisation.UID, @@ -41,7 +53,7 @@ func (iu *InviteUserService) Run(ctx context.Context) (*datastore.OrganisationIn UpdatedAt: time.Now(), } - err := iu.InviteRepo.CreateOrganisationInvite(ctx, iv) + err = iu.InviteRepo.CreateOrganisationInvite(ctx, iv) if err != nil { errMsg := "failed to invite member" log.FromContext(ctx).WithError(err).Error(errMsg) diff --git a/services/invite_user_test.go b/services/invite_user_test.go index 03df075918..20a82207d2 100644 --- a/services/invite_user_test.go +++ b/services/invite_user_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/auth" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" @@ -18,6 +20,7 @@ func TestInviteUserService(t *testing.T) { type args struct { inviteRepo datastore.OrganisationInviteRepository queue queue.Queuer + Licenser license.Licenser } dbErr := errors.New("failed to create invite") @@ -37,6 +40,9 @@ func TestInviteUserService(t *testing.T) { user: &datastore.User{}, organisation: &datastore.Organisation{}, mockDep: func(a args) { + licenser, _ := a.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + ivRepo, _ := a.inviteRepo.(*mocks.MockOrganisationInviteRepository) ivRepo.EXPECT().CreateOrganisationInvite( gomock.Any(), @@ -54,6 +60,9 @@ func TestInviteUserService(t *testing.T) { organisation: &datastore.Organisation{}, err: dbErr, mockDep: func(a args) { + licenser, _ := a.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + ivRepo, _ := a.inviteRepo.(*mocks.MockOrganisationInviteRepository) ivRepo.EXPECT().CreateOrganisationInvite( gomock.Any(), @@ -61,6 +70,17 @@ func TestInviteUserService(t *testing.T) { ).Return(dbErr) }, }, + { + name: "should_fail_to_invite_user_for_license_check", + inviteeEmail: "sidemen@default.com", + user: &datastore.User{}, + organisation: &datastore.Organisation{}, + err: ErrUserLimit, + mockDep: func(a args) { + licenser, _ := a.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(false, nil) + }, + }, } for _, tt := range tests { @@ -74,6 +94,7 @@ func TestInviteUserService(t *testing.T) { args := args{ inviteRepo: mocks.NewMockOrganisationInviteRepository(ctrl), queue: mocks.NewMockQueuer(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), } if tt.mockDep != nil { @@ -86,6 +107,7 @@ func TestInviteUserService(t *testing.T) { InviteeEmail: tt.inviteeEmail, User: tt.user, Organisation: tt.organisation, + Licenser: args.Licenser, Role: tt.role, } diff --git a/services/process_invite.go b/services/process_invite.go index 9ab0d79d6e..9b0b02b7aa 100644 --- a/services/process_invite.go +++ b/services/process_invite.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/api/models" "github.com/oklog/ulid/v2" @@ -21,13 +23,25 @@ type ProcessInviteService struct { UserRepo datastore.UserRepository OrgRepo datastore.OrganisationRepository OrgMemberRepo datastore.OrganisationMemberRepository + Licenser license.Licenser Token string Accepted bool NewUser *models.User } +var ErrUserLimit = errors.New("your instance has reached it's user limit, upgrade to add new users") + func (pis *ProcessInviteService) Run(ctx context.Context) error { + ok, err := pis.Licenser.CreateUser(ctx) + if err != nil { + return &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return &ServiceError{ErrMsg: ErrUserLimit.Error()} + } + iv, err := pis.InviteRepo.FetchOrganisationInviteByToken(ctx, pis.Token) if err != nil { log.FromContext(ctx).WithError(err).Error("failed to fetch organisation member invite by token and email") diff --git a/services/process_invite_test.go b/services/process_invite_test.go index 4d297ace2c..b994318d7d 100644 --- a/services/process_invite_test.go +++ b/services/process_invite_test.go @@ -23,6 +23,7 @@ func provideProcessInviteService(ctrl *gomock.Controller, token string, accepted UserRepo: mocks.NewMockUserRepository(ctrl), OrgRepo: mocks.NewMockOrganisationRepository(ctrl), OrgMemberRepo: mocks.NewMockOrganisationMemberRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), Token: token, Accepted: accepted, @@ -100,6 +101,9 @@ func TestProcessInviteService_Run(t *testing.T) { om, _ := pis.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -128,10 +132,29 @@ func TestProcessInviteService_Run(t *testing.T) { }, nil, ) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "organisation member invite already accepted", }, + + { + name: "should_error_for_licence_cant_user", + args: args{ + ctx: ctx, + token: "abcdef", + accepted: true, + newUser: nil, + }, + dbFn: func(pis *ProcessInviteService) { + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrMsg: ErrUserLimit.Error(), + }, { name: "should_error_for_invite_already_declined", args: args{ @@ -156,6 +179,9 @@ func TestProcessInviteService_Run(t *testing.T) { }, nil, ) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "organisation member invite already declined", @@ -185,6 +211,9 @@ func TestProcessInviteService_Run(t *testing.T) { }, nil, ) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "organisation member invite already expired", @@ -201,6 +230,9 @@ func TestProcessInviteService_Run(t *testing.T) { oir, _ := pis.InviteRepo.(*mocks.MockOrganisationInviteRepository) oir.EXPECT().FetchOrganisationInviteByToken(gomock.Any(), "abcdef"). Times(1).Return(nil, errors.New("failed")) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to fetch organisation member invite", @@ -241,6 +273,8 @@ func TestProcessInviteService_Run(t *testing.T) { Endpoint: "", }, }).Times(1).Return(nil) + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -272,6 +306,9 @@ func TestProcessInviteService_Run(t *testing.T) { u, _ := pis.UserRepo.(*mocks.MockUserRepository) u.EXPECT().FindUserByEmail(gomock.Any(), "test@email.com"). Times(1).Return(nil, errors.New("failed")) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to find user by email", @@ -332,6 +369,9 @@ func TestProcessInviteService_Run(t *testing.T) { om, _ := pis.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: false, }, @@ -363,6 +403,9 @@ func TestProcessInviteService_Run(t *testing.T) { u, _ := pis.UserRepo.(*mocks.MockUserRepository) u.EXPECT().FindUserByEmail(gomock.Any(), "test@email.com"). Times(1).Return(nil, datastore.ErrUserNotFound) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "new user is nil", @@ -401,6 +444,9 @@ func TestProcessInviteService_Run(t *testing.T) { u, _ := pis.UserRepo.(*mocks.MockUserRepository) u.EXPECT().FindUserByEmail(gomock.Any(), "test@email.com"). Times(1).Return(nil, datastore.ErrUserNotFound) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "first_name:please provide a first name", @@ -441,6 +487,9 @@ func TestProcessInviteService_Run(t *testing.T) { Times(1).Return(nil, datastore.ErrUserNotFound) u.EXPECT().CreateUser(gomock.Any(), gomock.Any()).Times(1).Return(errors.New("failed")) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to create user", @@ -482,6 +531,8 @@ func TestProcessInviteService_Run(t *testing.T) { o, _ := pis.OrgRepo.(*mocks.MockOrganisationRepository) o.EXPECT().FetchOrganisationByID(gomock.Any(), "123ab"). Times(1).Return(nil, errors.New("failed")) + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to fetch organisation by id", @@ -588,6 +639,9 @@ func TestProcessInviteService_Run(t *testing.T) { om, _ := pis.OrgMemberRepo.(*mocks.MockOrganisationMemberRepository) om.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := pis.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrMsg: "failed to update accepted organisation invite", diff --git a/services/project_service.go b/services/project_service.go index e9bc41c761..ca2b6c9a53 100644 --- a/services/project_service.go +++ b/services/project_service.go @@ -7,6 +7,8 @@ import ( "net/http" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/auth" "github.com/oklog/ulid/v2" @@ -22,20 +24,33 @@ type ProjectService struct { projectRepo datastore.ProjectRepository eventRepo datastore.EventRepository eventDeliveryRepo datastore.EventDeliveryRepository + Licenser license.Licenser cache cache.Cache } -func NewProjectService(apiKeyRepo datastore.APIKeyRepository, projectRepo datastore.ProjectRepository, eventRepo datastore.EventRepository, eventDeliveryRepo datastore.EventDeliveryRepository, cache cache.Cache) (*ProjectService, error) { +func NewProjectService(apiKeyRepo datastore.APIKeyRepository, projectRepo datastore.ProjectRepository, eventRepo datastore.EventRepository, eventDeliveryRepo datastore.EventDeliveryRepository, licenser license.Licenser, cache cache.Cache) (*ProjectService, error) { return &ProjectService{ apiKeyRepo: apiKeyRepo, projectRepo: projectRepo, eventRepo: eventRepo, eventDeliveryRepo: eventDeliveryRepo, + Licenser: licenser, cache: cache, }, nil } +var ErrProjectLimit = errors.New("your instance has reached it's project limit, upgrade to create more projects") + func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models.CreateProject, org *datastore.Organisation, member *datastore.OrganisationMember) (*datastore.Project, *models.APIKeyResponse, error) { + ok, err := ps.Licenser.CreateProject(ctx) + if err != nil { + return nil, nil, util.NewServiceError(http.StatusBadRequest, err) + } + + if !ok { + return nil, nil, util.NewServiceError(http.StatusBadRequest, ErrProjectLimit) + } + projectName := newProject.Name projectConfig := newProject.Config.Transform() @@ -84,7 +99,7 @@ func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models. UpdatedAt: time.Now(), } - err := ps.projectRepo.CreateProject(ctx, project) + err = ps.projectRepo.CreateProject(ctx, project) if err != nil { log.FromContext(ctx).WithError(err).Error("failed to create project") if errors.Is(err, datastore.ErrDuplicateProjectName) { @@ -129,6 +144,11 @@ func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models. Key: keyString, } + // if this is a community license, add this project to list of enabled projects + // because if the initial license check above passed, then the project count limit had + // not been reached + ps.Licenser.AddEnabledProject(project.UID) + return project, resp, nil } diff --git a/services/project_service_test.go b/services/project_service_test.go index 5b59b4f418..deda3d055b 100644 --- a/services/project_service_test.go +++ b/services/project_service_test.go @@ -21,9 +21,10 @@ func provideProjectService(ctrl *gomock.Controller) (*ProjectService, error) { eventRepo := mocks.NewMockEventRepository(ctrl) eventDeliveryRepo := mocks.NewMockEventDeliveryRepository(ctrl) apiKeyRepo := mocks.NewMockAPIKeyRepository(ctrl) + l := mocks.NewMockLicenser(ctrl) cache := mocks.NewMockCache(ctrl) - return NewProjectService(apiKeyRepo, projectRepo, eventRepo, eventDeliveryRepo, cache) + return NewProjectService(apiKeyRepo, projectRepo, eventRepo, eventDeliveryRepo, l, cache) } func TestProjectService_CreateProject(t *testing.T) { @@ -83,6 +84,10 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) }, wantProject: &datastore.Project{ Name: "test_project", @@ -150,6 +155,10 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) }, wantProject: &datastore.Project{ Name: "test_project", @@ -202,6 +211,10 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) }, wantProject: &datastore.Project{ Name: "test_project_1", @@ -255,6 +268,10 @@ func TestProjectService_CreateProject(t *testing.T) { apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) }, wantProject: &datastore.Project{ Name: "test_project", @@ -309,6 +326,9 @@ func TestProjectService_CreateProject(t *testing.T) { a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().CreateProject(gomock.Any(), gomock.Any()). Times(1).Return(errors.New("failed")) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrCode: http.StatusBadRequest, @@ -380,11 +400,45 @@ func TestProjectService_CreateProject(t *testing.T) { a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().CreateProject(gomock.Any(), gomock.Any()). Times(1).Return(datastore.ErrDuplicateProjectName) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) }, wantErr: true, wantErrCode: http.StatusBadRequest, wantErrMsg: "a project with this name already exists", }, + { + name: "should_error_for_cant_create_project", + args: args{ + ctx: ctx, + newProject: &models.CreateProject{ + Name: "test_project", + Type: "incoming", + LogoURL: "https://google.com", + Config: &models.ProjectConfig{ + Signature: &models.SignatureConfiguration{ + Header: "X-Convoy-Signature", + }, + Strategy: &models.StrategyConfiguration{ + Type: "linear", + Duration: 20, + RetryCount: 4, + }, + ReplayAttacks: true, + }, + }, + org: &datastore.Organisation{UID: "1234"}, + member: &datastore.OrganisationMember{}, + }, + dbFn: func(gs *ProjectService) { + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrCode: http.StatusBadRequest, + wantErrMsg: ErrProjectLimit.Error(), + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/services/register_user.go b/services/register_user.go index 0ca3ccc56a..aa989cd585 100644 --- a/services/register_user.go +++ b/services/register_user.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" - "github.com/frain-dev/convoy/pkg/msgpack" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/pkg/msgpack" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/internal/email" "github.com/frain-dev/convoy/pkg/log" @@ -26,12 +28,22 @@ type RegisterUserService struct { Queue queue.Queuer JWT *jwt.Jwt ConfigRepo datastore.ConfigurationRepository + Licenser license.Licenser BaseURL string Data *models.RegisterUser } func (u *RegisterUserService) Run(ctx context.Context) (*datastore.User, *jwt.Token, error) { + ok, err := u.Licenser.CreateUser(ctx) + if err != nil { + return nil, nil, &ServiceError{ErrMsg: err.Error()} + } + + if !ok { + return nil, nil, &ServiceError{ErrMsg: ErrUserLimit.Error()} + } + config, err := u.ConfigRepo.LoadConfiguration(ctx) if err != nil && !errors.Is(err, datastore.ErrConfigNotFound) { return nil, nil, &ServiceError{ErrMsg: "failed to load configuration", Err: err} @@ -77,13 +89,16 @@ func (u *RegisterUserService) Run(ctx context.Context) (*datastore.User, *jwt.To co := CreateOrganisationService{ OrgRepo: u.OrgRepo, OrgMemberRepo: u.OrgMemberRepo, + Licenser: u.Licenser, NewOrg: &models.Organisation{Name: u.Data.OrganisationName}, User: user, } _, err = co.Run(ctx) if err != nil { - return nil, nil, err + if !errors.Is(err, ErrOrgLimit) && !errors.Is(err, ErrUserLimit) { + return nil, nil, err + } } token, err := u.JWT.GenerateToken(user) diff --git a/services/register_user_test.go b/services/register_user_test.go index 63604d6060..9cb6b87576 100644 --- a/services/register_user_test.go +++ b/services/register_user_test.go @@ -25,6 +25,7 @@ func provideRegisterUserService(ctrl *gomock.Controller, t *testing.T, baseUrl s OrgRepo: mocks.NewMockOrganisationRepository(ctrl), OrgMemberRepo: mocks.NewMockOrganisationMemberRepository(ctrl), Queue: mocks.NewMockQueuer(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), JWT: jwt.NewJwt(&configuration.Auth.Jwt, c), ConfigRepo: mocks.NewMockConfigurationRepository(ctrl), BaseURL: baseUrl, @@ -86,8 +87,39 @@ func TestRegisterUserService_Run(t *testing.T) { orgMemberRepo.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) queue.EXPECT().Write(gomock.Any(), gomock.Any(), gomock.Any()) + + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, }, + { + name: "should_not_register_user_when_cannot_create_user", + wantConfig: true, + args: args{ + ctx: ctx, + user: &models.RegisterUser{ + FirstName: "test", + LastName: "test", + Email: "test@test.com", + Password: "123456", + OrganisationName: "test", + }, + }, + wantUser: &datastore.User{ + UID: "12345", + FirstName: "test", + LastName: "test", + Email: "test@test.com", + }, + dbFn: func(u *RegisterUserService) { + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(false, nil) + }, + wantErr: true, + wantErrMsg: ErrUserLimit.Error(), + }, + { name: "should_fail_to_load_config", wantConfig: true, @@ -102,6 +134,9 @@ func TestRegisterUserService_Run(t *testing.T) { }, }, dbFn: func(u *RegisterUserService) { + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + configRepo, _ := u.ConfigRepo.(*mocks.MockConfigurationRepository) configRepo.EXPECT().LoadConfiguration(gomock.Any()).Times(1).Return(nil, errors.New("failed")) }, @@ -142,6 +177,10 @@ func TestRegisterUserService_Run(t *testing.T) { orgMemberRepo.EXPECT().CreateOrganisationMember(gomock.Any(), gomock.Any()).Times(1).Return(nil) queue.EXPECT().Write(gomock.Any(), gomock.Any(), gomock.Any()) + + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateOrg(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) }, }, { @@ -158,6 +197,9 @@ func TestRegisterUserService_Run(t *testing.T) { }, }, dbFn: func(u *RegisterUserService) { + licenser, _ := u.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateUser(gomock.Any()).Times(1).Return(true, nil) + configRepo, _ := u.ConfigRepo.(*mocks.MockConfigurationRepository) configRepo.EXPECT().LoadConfiguration(gomock.Any()).Times(1).Return(&datastore.Configuration{ UID: "12345", diff --git a/services/update_endpoint.go b/services/update_endpoint.go index 001a3aed9f..a0381cbad9 100644 --- a/services/update_endpoint.go +++ b/services/update_endpoint.go @@ -4,6 +4,9 @@ import ( "context" "time" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/pkg/log" "github.com/frain-dev/convoy/api/models" @@ -16,6 +19,7 @@ type UpdateEndpointService struct { Cache cache.Cache EndpointRepo datastore.EndpointRepository ProjectRepo datastore.ProjectRepository + Licenser license.Licenser E models.UpdateEndpoint Endpoint *datastore.Endpoint @@ -37,7 +41,7 @@ func (a *UpdateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e return nil, &ServiceError{ErrMsg: err.Error()} } - endpoint, err = updateEndpoint(endpoint, a.E, a.Project) + endpoint, err = a.updateEndpoint(endpoint, a.E, a.Project) if err != nil { return nil, &ServiceError{ErrMsg: err.Error()} } @@ -52,7 +56,7 @@ func (a *UpdateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e return endpoint, nil } -func updateEndpoint(endpoint *datastore.Endpoint, e models.UpdateEndpoint, project *datastore.Project) (*datastore.Endpoint, error) { +func (a *UpdateEndpointService) updateEndpoint(endpoint *datastore.Endpoint, e models.UpdateEndpoint, project *datastore.Project) (*datastore.Endpoint, error) { endpoint.Url = e.URL endpoint.Description = e.Description @@ -80,6 +84,11 @@ func updateEndpoint(endpoint *datastore.Endpoint, e models.UpdateEndpoint, proje if e.HttpTimeout != 0 { endpoint.HttpTimeout = e.HttpTimeout + + if !a.Licenser.AdvancedEndpointMgmt() { + // switch to default timeout + endpoint.HttpTimeout = convoy.HTTP_TIMEOUT + } } if !util.IsStringEmpty(e.OwnerID) { diff --git a/services/update_endpoint_test.go b/services/update_endpoint_test.go index 0f5a452ad6..58d6801246 100644 --- a/services/update_endpoint_test.go +++ b/services/update_endpoint_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/mocks" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -18,6 +20,7 @@ func provideUpdateEndpointService(ctrl *gomock.Controller, e models.UpdateEndpoi Cache: mocks.NewMockCache(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), ProjectRepo: mocks.NewMockProjectRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), E: e, Endpoint: Endpoint, Project: Project, @@ -71,6 +74,9 @@ func TestUpdateEndpointService_Run(t *testing.T) { a.EXPECT().UpdateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()). Times(1).Return(nil) + + licenser, _ := as.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantErr: false, }, @@ -96,10 +102,51 @@ func TestUpdateEndpointService_Run(t *testing.T) { a.EXPECT().UpdateEndpoint(gomock.Any(), gomock.Any(), gomock.Any()). Times(1).Return(errors.New("failed")) + + licenser, _ := as.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, wantErr: true, wantErrMsg: "an error occurred while updating endpoints", }, + { + name: "should_default_endpoint_http_timeout_for_license_check_failed", + args: args{ + ctx: ctx, + e: models.UpdateEndpoint{ + Name: stringPtr("Endpoint2"), + Description: "test_endpoint", + URL: "https://www.google.com/webhp", + RateLimit: 10000, + RateLimitDuration: 60, + HttpTimeout: 200, + }, + endpoint: &datastore.Endpoint{UID: "endpoint2"}, + project: project, + }, + wantEndpoint: &datastore.Endpoint{ + Name: "Endpoint2", + Description: "test_endpoint", + Url: "https://www.google.com/webhp", + RateLimit: 10000, + RateLimitDuration: 60, + HttpTimeout: convoy.HTTP_TIMEOUT, + }, + dbFn: func(as *UpdateEndpointService) { + a, _ := as.EndpointRepo.(*mocks.MockEndpointRepository) + a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), "1234567890"). + Times(1).Return(&datastore.Endpoint{UID: "endpoint2"}, nil) + + a.EXPECT().UpdateEndpoint(gomock.Any(), gomock.Cond(func(x any) bool { + endpoint := x.(*datastore.Endpoint) + return endpoint.HttpTimeout == convoy.HTTP_TIMEOUT + }), gomock.Any()).Times(1).Return(nil) + + licenser, _ := as.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) + }, + wantErr: false, + }, { name: "should_error_for_endpoint_not_found", args: args{ diff --git a/services/update_subscription.go b/services/update_subscription.go index c94a41e343..236323d980 100644 --- a/services/update_subscription.go +++ b/services/update_subscription.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "errors" + + "github.com/frain-dev/convoy/internal/pkg/license" "gopkg.in/guregu/null.v4" "github.com/frain-dev/convoy/api/models" @@ -18,9 +20,11 @@ var ( ) type UpdateSubscriptionService struct { - SubRepo datastore.SubscriptionRepository - EndpointRepo datastore.EndpointRepository - SourceRepo datastore.SourceRepository + SubRepo datastore.SubscriptionRepository + EndpointRepo datastore.EndpointRepository + SourceRepo datastore.SourceRepository + Licenser license.Licenser + ProjectId string SubscriptionId string Update *models.UpdateSubscription @@ -46,7 +50,7 @@ func (s *UpdateSubscriptionService) Run(ctx context.Context) (*datastore.Subscri subscription.SourceID = s.Update.SourceID } - if !util.IsStringEmpty(s.Update.Function) { + if !util.IsStringEmpty(s.Update.Function) && s.Licenser.Transformations() { subscription.Function = null.StringFrom(s.Update.Function) } @@ -102,7 +106,7 @@ func (s *UpdateSubscriptionService) Run(ctx context.Context) (*datastore.Subscri subscription.RetryConfig.RetryCount = s.Update.RetryConfig.RetryCount } - if s.Update.FilterConfig != nil { + if s.Update.FilterConfig != nil && s.Licenser.AdvancedSubscriptions() { if len(s.Update.FilterConfig.EventTypes) > 0 { subscription.FilterConfig.EventTypes = s.Update.FilterConfig.EventTypes } diff --git a/services/update_subscription_test.go b/services/update_subscription_test.go index 64fc5c4455..0dcdfb942a 100644 --- a/services/update_subscription_test.go +++ b/services/update_subscription_test.go @@ -18,6 +18,7 @@ func provideUpdateSubscriptionService(ctrl *gomock.Controller, projectID string, SubRepo: mocks.NewMockSubscriptionRepository(ctrl), EndpointRepo: mocks.NewMockEndpointRepository(ctrl), SourceRepo: mocks.NewMockSourceRepository(ctrl), + Licenser: mocks.NewMockLicenser(ctrl), ProjectId: projectID, SubscriptionId: subID, Update: update, diff --git a/testcon/docker_e2e_integration_test.go b/testcon/docker_e2e_integration_test.go index e7533cd84d..fee94a384a 100644 --- a/testcon/docker_e2e_integration_test.go +++ b/testcon/docker_e2e_integration_test.go @@ -5,14 +5,16 @@ package testcon import ( "context" + "os" + "strings" + "testing" + "time" + "github.com/docker/compose/v2/pkg/api" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" tc "github.com/testcontainers/testcontainers-go/modules/compose" "github.com/testcontainers/testcontainers-go/wait" - "strings" - "testing" - "time" ) type DockerE2EIntegrationTestSuite struct { @@ -34,11 +36,16 @@ func (d *DockerE2EIntegrationTestSuite) SetupSuite() { t.Cleanup(cancel) // ignore ryuk error - err = compose.WaitForService("postgres", wait.NewLogStrategy("ready").WithStartupTimeout(60*time.Second)). + err = compose. + WithEnv(map[string]string{ + "CONVOY_LICENSE_KEY": os.Getenv("TEST_LICENSE_KEY"), + }). + WaitForService("postgres", wait.NewLogStrategy("ready").WithStartupTimeout(60*time.Second)). WaitForService("redis_server", wait.NewLogStrategy("Ready to accept connections").WithStartupTimeout(10*time.Second)). WaitForService("migrate", wait.NewLogStrategy("migration up succeeded").WithStartupTimeout(60*time.Second)). Up(ctx, tc.Wait(true), tc.WithRecreate(api.RecreateNever)) - if err != nil && !strings.Contains(err.Error(), "Ryuk") { + + if err != nil && !strings.Contains(err.Error(), "Ryuk") && !strings.Contains(err.Error(), "container exited with code 0") { require.NoError(t, err) } @@ -46,11 +53,9 @@ func (d *DockerE2EIntegrationTestSuite) SetupSuite() { } func (d *DockerE2EIntegrationTestSuite) SetupTest() { - } func (d *DockerE2EIntegrationTestSuite) TearDownTest() { - } func TestDockerE2EIntegrationTestSuite(t *testing.T) { diff --git a/testcon/docker_e2e_integration_test_helper.go b/testcon/docker_e2e_integration_test_helper.go index 31cc07a477..9bdf189c0f 100644 --- a/testcon/docker_e2e_integration_test_helper.go +++ b/testcon/docker_e2e_integration_test_helper.go @@ -7,6 +7,16 @@ import ( "context" "errors" "fmt" + "io" + "net" + "net/http" + "os" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + convoy "github.com/frain-dev/convoy-go/v2" "github.com/frain-dev/convoy/api/testdb" "github.com/frain-dev/convoy/auth" @@ -21,19 +31,12 @@ import ( "github.com/frain-dev/convoy/testcon/manifest" "github.com/oklog/ulid/v2" "github.com/stretchr/testify/require" - "io" - "net" - "net/http" - "os" - "strconv" - "sync" - "sync/atomic" - "testing" - "time" ) -var once sync.Once -var pDB *postgres.Postgres +var ( + once sync.Once + pDB *postgres.Postgres +) func getConfig() config.Configuration { err := config.LoadConfig("./testdata/convoy-host.json") diff --git a/testcon/testdata/docker-compose-test.yml b/testcon/testdata/docker-compose-test.yml index cd916f8126..d2b80d0a1a 100644 --- a/testcon/testdata/docker-compose-test.yml +++ b/testcon/testdata/docker-compose-test.yml @@ -10,6 +10,8 @@ services: context: ../../ dockerfile: Dockerfile.dev command: [ "/start.sh" ] + environment: + - CONVOY_LICENSE_KEY volumes: - ./convoy-docker.json:/convoy.json restart: on-failure @@ -37,6 +39,8 @@ services: context: ../../ dockerfile: Dockerfile.dev entrypoint: ["./cmd", "agent", "--config", "convoy.json"] + environment: + - CONVOY_LICENSE_KEY volumes: - ./convoy-docker.json:/convoy.json restart: on-failure diff --git a/web/ui/dashboard/src/app/components/tag/tag.component.ts b/web/ui/dashboard/src/app/components/tag/tag.component.ts index 7249bd83bc..a6c85d9248 100644 --- a/web/ui/dashboard/src/app/components/tag/tag.component.ts +++ b/web/ui/dashboard/src/app/components/tag/tag.component.ts @@ -9,11 +9,10 @@ import { STATUS_COLOR } from 'src/app/models/global.model'; template: ` `, - host: { class: 'rounded-22px w-fit text-center text-12 justify-between gap-x-4px disabled:opacity-50', '[class]': 'classes' } + host: { class: 'rounded-22px w-fit text-center text-12 justify-between gap-x-4px disabled:opacity-50 flex items-center justify-center', '[class]': 'classes' } }) export class TagComponent implements OnInit { @Input('className') class!: string; - @Input('fill') fill: 'outline' | 'soft' | 'solid' | 'soft-outline' = 'soft'; @Input('color') color: 'primary' | 'error' | 'success' | 'warning' | 'neutral' = 'neutral'; @Input('size') size: 'sm' | 'md' | 'lg' = 'md'; diff --git a/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html b/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html index 1adda9d2d5..cac80ee125 100644 --- a/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html +++ b/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html @@ -160,10 +160,18 @@
-

- Endpoint Timeout - How many seconds should Convoy wait for a response from this endpoint before timing out? -

+
+

+ Endpoint Timeout + How many seconds should Convoy wait for a response from this endpoint before timing out? +

+
+ + + + Business +
+
+
diff --git a/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts b/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts index 7c7cd3e8bb..093049b542 100644 --- a/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts +++ b/web/ui/dashboard/src/app/private/components/create-source/create-source.component.ts @@ -6,6 +6,7 @@ import { GeneralService } from 'src/app/services/general/general.service'; import { PrivateService } from '../../private.service'; import { CreateSourceService } from './create-source.service'; import { RbacService } from 'src/app/services/rbac/rbac.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-create-source', @@ -175,7 +176,7 @@ export class CreateSourceComponent implements OnInit { sourceURL!: string; showTransformDialog = false; - constructor(private formBuilder: FormBuilder, private createSourceService: CreateSourceService, public privateService: PrivateService, private route: ActivatedRoute, private router: Router, private generalService: GeneralService) {} + constructor(private formBuilder: FormBuilder, private createSourceService: CreateSourceService, public privateService: PrivateService, private route: ActivatedRoute, private router: Router, private generalService: GeneralService, public licenseService: LicensesService) {} async ngOnInit() { if (this.privateService.getProjectDetails.type === 'incoming') diff --git a/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts b/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts index 71990fc8f7..7b22aee15a 100644 --- a/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts +++ b/web/ui/dashboard/src/app/private/components/create-source/create-source.module.ts @@ -19,6 +19,7 @@ import { NotificationComponent } from 'src/app/components/notification/notificat import { ConfigButtonComponent } from '../config-button/config-button.component'; import { SourceURLComponent } from './source-url/source-url.component'; import { CreateTransformFunctionComponent } from '../create-transform-function/create-transform-function.component'; +import { TagComponent } from 'src/app/components/tag/tag.component'; @NgModule({ declarations: [CreateSourceComponent], @@ -45,7 +46,8 @@ import { CreateTransformFunctionComponent } from '../create-transform-function/c NotificationComponent, ConfigButtonComponent, SourceURLComponent, - CreateTransformFunctionComponent + CreateTransformFunctionComponent, + TagComponent ], exports: [CreateSourceComponent] }) diff --git a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html index be5c241fae..93885e912e 100644 --- a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html +++ b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.html @@ -144,11 +144,19 @@

Webhook Subscription Configuration

-

Events Filter

+
+

Events Filter

+
+ + + + Business +
+

Filter events received by request body and header.

- +
@@ -164,11 +172,21 @@

Events Filter

-

Transform

+
+
+

Transform

+
+
+ + + + Business +
+

Transform request body of events with a javascript function.

- +
diff --git a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts index c613b1d4ad..f19972d426 100644 --- a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts +++ b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.component.ts @@ -9,6 +9,7 @@ import { CreateSourceComponent } from '../create-source/create-source.component' import { CreateSubscriptionService } from './create-subscription.service'; import { RbacService } from 'src/app/services/rbac/rbac.service'; import { SUBSCRIPTION } from 'src/app/models/subscription'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-create-subscription', @@ -78,7 +79,7 @@ export class CreateSubscriptionComponent implements OnInit { subscription!: SUBSCRIPTION; currentRoute = window.location.pathname.split('/').reverse()[0]; - constructor(private formBuilder: FormBuilder, private privateService: PrivateService, private createSubscriptionService: CreateSubscriptionService, private route: ActivatedRoute, private router: Router) {} + constructor(private formBuilder: FormBuilder, private privateService: PrivateService, private createSubscriptionService: CreateSubscriptionService, private route: ActivatedRoute, private router: Router, public licenseService: LicensesService) {} async ngOnInit() { this.isLoadingForm = true; diff --git a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts index d60ce86957..c32b47a2dd 100644 --- a/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts +++ b/web/ui/dashboard/src/app/private/components/create-subscription/create-subscription.module.ts @@ -22,6 +22,7 @@ import { NotificationComponent } from 'src/app/components/notification/notificat import { CreateTransformFunctionComponent } from '../create-transform-function/create-transform-function.component'; import { ConfigButtonComponent } from '../config-button/config-button.component'; import { SourceURLComponent } from '../create-source/source-url/source-url.component'; +import { TagComponent } from 'src/app/components/tag/tag.component'; @NgModule({ declarations: [CreateSubscriptionComponent], @@ -51,7 +52,8 @@ import { SourceURLComponent } from '../create-source/source-url/source-url.compo NotificationComponent, CreateTransformFunctionComponent, ConfigButtonComponent, - SourceURLComponent + SourceURLComponent, + TagComponent ], exports: [CreateSubscriptionComponent] }) diff --git a/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.html b/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.html index 7f799d12ae..100486b6ef 100644 --- a/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.html +++ b/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.html @@ -1,148 +1,157 @@ - -
- -
-
-
- - - - - Create Portal Link - +
+
+ + + + + Create Portal Link + -
-
-
-
-
{{ link.name }}
-
- {{ link.endpoints_metadata.length }} Endpoint{{ link.endpoints_metadata.length > 1 ? 's' : '' }} +
+
+
+
+
{{ link.name }}
+
+ {{ link.endpoints_metadata.length }} Endpoint{{ link.endpoints_metadata.length > 1 ? 's' : '' }} +
-
-
- +
+ -
    -
  • - -
    • -
  • - -
    • -
+
    +
  • + +
    • +
  • + +
    • +
+
-
-
- -
- {{ link.url }} - - - - - - -
- -
- + +
-
- + - - + + +
+ +
+
diff --git a/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts b/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts index 828fca4bcc..bd20cba085 100644 --- a/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts +++ b/web/ui/dashboard/src/app/private/pages/project/portal-links/portal-links.component.ts @@ -24,6 +24,7 @@ import { PermissionDirective } from 'src/app/private/components/permission/permi import { LoaderModule } from 'src/app/private/components/loader/loader.module'; import { EndpointFilterComponent } from 'src/app/private/components/endpoints-filter/endpoints-filter.component'; import { TagComponent } from 'src/app/components/tag/tag.component'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-portal-links', @@ -70,10 +71,10 @@ export class PortalLinksComponent implements OnInit { @ViewChild('linksEndpointFilter', { static: true }) linksEndpointFilter!: ElementRef; linksEndpointFilter$!: Observable; - constructor(public privateService: PrivateService, public router: Router, private portalLinksService: PortalLinksService, private route: ActivatedRoute, private generalService: GeneralService) {} + constructor(public privateService: PrivateService, public router: Router, private portalLinksService: PortalLinksService, private route: ActivatedRoute, private generalService: GeneralService, public licenseService: LicensesService) {} ngOnInit() { - this.getPortalLinks(); + if(this.licenseService.hasLicense('PORTAL_LINKS')) this.getPortalLinks(); const urlParam = this.route.snapshot.params.id; if (urlParam) { diff --git a/web/ui/dashboard/src/app/private/pages/project/project.component.html b/web/ui/dashboard/src/app/private/pages/project/project.component.html index a0ea920165..59a1d2109b 100644 --- a/web/ui/dashboard/src/app/private/pages/project/project.component.html +++ b/web/ui/dashboard/src/app/private/pages/project/project.component.html @@ -27,7 +27,15 @@
    -
  • - +
    + + + + Business +
diff --git a/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts b/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts index bf3444b8a5..2f0bf11056 100644 --- a/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts +++ b/web/ui/dashboard/src/app/private/pages/settings/settings.component.ts @@ -1,6 +1,7 @@ -import {Location} from '@angular/common'; -import {Component, OnInit} from '@angular/core'; -import {ActivatedRoute, Router} from '@angular/router'; +import { Location } from '@angular/common'; +import { Component, OnInit } from '@angular/core'; +import { ActivatedRoute, Router } from '@angular/router'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; export type SETTINGS = 'organisation settings' | 'configuration settings' | 'personal access tokens' | 'team'; @@ -13,14 +14,14 @@ export class SettingsComponent implements OnInit { activePage: SETTINGS = 'organisation settings'; settingsMenu: { name: SETTINGS; icon: string; svg: 'stroke' | 'fill' }[] = [ { name: 'organisation settings', icon: 'org', svg: 'fill' }, - { name: 'team', icon: 'team', svg: 'stroke' }, + { name: 'team', icon: 'team', svg: 'stroke' } // { name: 'configuration settings', icon: 'settings', svg: 'fill' } ]; - constructor(private router: Router, private route: ActivatedRoute, private location: Location) {} + constructor(private router: Router, private route: ActivatedRoute, public licenseService: LicensesService) {} ngOnInit() { - this.toggleActivePage(this.route.snapshot.queryParams?.activePage ?? 'organisation settings'); + if (this.licenseService.hasLicense('CREATE_ORG_MEMBER')) this.toggleActivePage(this.route.snapshot.queryParams?.activePage ?? 'organisation settings'); } toggleActivePage(activePage: SETTINGS) { diff --git a/web/ui/dashboard/src/app/private/private.component.html b/web/ui/dashboard/src/app/private/private.component.html index 024c2037df..ed44f8f909 100644 --- a/web/ui/dashboard/src/app/private/private.component.html +++ b/web/ui/dashboard/src/app/private/private.component.html @@ -46,11 +46,17 @@ -
  • +
  • +
    + + + + Business +
    • diff --git a/web/ui/dashboard/src/app/private/private.component.ts b/web/ui/dashboard/src/app/private/private.component.ts index 00862ebfaf..3602394a6d 100644 --- a/web/ui/dashboard/src/app/private/private.component.ts +++ b/web/ui/dashboard/src/app/private/private.component.ts @@ -7,6 +7,7 @@ import { FormBuilder, FormControl, FormGroup, Validators } from '@angular/forms' import { JwtHelperService } from '@auth0/angular-jwt'; import { differenceInSeconds } from 'date-fns'; import { Observable, Subscription } from 'rxjs'; +import { LicensesService } from '../services/licenses/licenses.service'; @Component({ selector: 'app-private', @@ -47,13 +48,13 @@ export class PrivateComponent implements OnInit { private jwtHelper: JwtHelperService = new JwtHelperService(); private shouldShowOrgSubscription: Subscription | undefined; - constructor(private generalService: GeneralService, public router: Router, public privateService: PrivateService, private formBuilder: FormBuilder) {} + constructor(private generalService: GeneralService, public router: Router, public privateService: PrivateService, private formBuilder: FormBuilder, public licenseService: LicensesService) {} async ngOnInit() { this.shouldShowOrgModal(); this.checkIfTokenIsExpired(); - await Promise.all([this.getConfiguration(), this.getUserDetails(), this.getOrganizations()]); + await Promise.all([this.getConfiguration(), this.licenseService.setLicenses(), this.getUserDetails(), this.getOrganizations()]); } ngOnDestroy() { @@ -71,9 +72,9 @@ export class PrivateComponent implements OnInit { return authDetails ? JSON.parse(authDetails) : false; } - shouldMountAppRouter(): boolean { - return !this.isLoadingOrganisations && (Boolean(this.organisations?.length) || this.router.url.startsWith('/user-settings')) - } + shouldMountAppRouter(): boolean { + return !this.isLoadingOrganisations && (Boolean(this.organisations?.length) || this.router.url.startsWith('/user-settings')); + } async getConfiguration() { try { @@ -162,6 +163,7 @@ export class PrivateComponent implements OnInit { this.generalService.showNotification({ style: 'success', message: response.message }); this.creatingOrganisation = false; this.dialog.nativeElement.close(); + this.licenseService.setLicenses(); await this.getOrganizations(true); this.selectOrganisation(response.data); diff --git a/web/ui/dashboard/src/app/public/login/login.component.html b/web/ui/dashboard/src/app/public/login/login.component.html index d9333a1a50..191952687f 100644 --- a/web/ui/dashboard/src/app/public/login/login.component.html +++ b/web/ui/dashboard/src/app/public/login/login.component.html @@ -30,7 +30,7 @@ - +
    diff --git a/web/ui/dashboard/src/app/public/login/login.component.ts b/web/ui/dashboard/src/app/public/login/login.component.ts index 07f0c97f2c..c31a83ee7a 100644 --- a/web/ui/dashboard/src/app/public/login/login.component.ts +++ b/web/ui/dashboard/src/app/public/login/login.component.ts @@ -9,6 +9,7 @@ import { LoaderModule } from 'src/app/private/components/loader/loader.module'; import { PrivateService } from 'src/app/private/private.service'; import { ORGANIZATION_DATA } from 'src/app/models/organisation.model'; import { SignupService } from '../signup/signup.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'app-login', @@ -29,10 +30,11 @@ export class LoginComponent implements OnInit { isSignupEnabled = false; organisations?: ORGANIZATION_DATA[]; - constructor(private formBuilder: FormBuilder, public router: Router, private loginService: LoginService, private signupService: SignupService, private privateService: PrivateService) {} + constructor(private formBuilder: FormBuilder, public router: Router, private loginService: LoginService, private signupService: SignupService, private privateService: PrivateService, public licenseService: LicensesService) {} ngOnInit() { this.getSignUpConfig(); + this.licenseService.setLicenses(); } async getSignUpConfig() { diff --git a/web/ui/dashboard/src/app/public/signup/signup.component.ts b/web/ui/dashboard/src/app/public/signup/signup.component.ts index 179ca11048..d0e1c708ce 100644 --- a/web/ui/dashboard/src/app/public/signup/signup.component.ts +++ b/web/ui/dashboard/src/app/public/signup/signup.component.ts @@ -7,6 +7,7 @@ import { InputDirective, InputErrorComponent, InputFieldDirective, LabelComponen import { LoaderModule } from 'src/app/private/components/loader/loader.module'; import { HubspotService } from 'src/app/services/hubspot/hubspot.service'; import { SignupService } from './signup.service'; +import { LicensesService } from 'src/app/services/licenses/licenses.service'; @Component({ selector: 'convoy-signup', @@ -27,9 +28,13 @@ export class SignupComponent implements OnInit { org_name: ['', Validators.required] }); - constructor(private formBuilder: FormBuilder, private signupService: SignupService, public router: Router, private hubspotService: HubspotService) {} + constructor(private formBuilder: FormBuilder, private signupService: SignupService, public router: Router, private hubspotService: HubspotService, private licenseService: LicensesService) {} - ngOnInit(): void {} + ngOnInit(): void { + this.licenseService.setLicenses(); + + if (!this.licenseService.hasLicense('CREATE_USER')) this.router.navigateByUrl('/login'); + } async signup() { if (this.signupForm.invalid) return this.signupForm.markAllAsTouched(); diff --git a/web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts b/web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts new file mode 100644 index 0000000000..13b7108200 --- /dev/null +++ b/web/ui/dashboard/src/app/services/licenses/licenses.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { LicensesService } from './licenses.service'; + +describe('LicensesService', () => { + let service: LicensesService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(LicensesService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/web/ui/dashboard/src/app/services/licenses/licenses.service.ts b/web/ui/dashboard/src/app/services/licenses/licenses.service.ts new file mode 100644 index 0000000000..0ed3a106a7 --- /dev/null +++ b/web/ui/dashboard/src/app/services/licenses/licenses.service.ts @@ -0,0 +1,51 @@ +import { Injectable } from '@angular/core'; +import { HttpService } from '../http/http.service'; +import { HTTP_RESPONSE } from 'src/app/models/global.model'; + +@Injectable({ + providedIn: 'root' +}) +export class LicensesService { + constructor(private http: HttpService) {} + + getLicenses(): Promise { + return new Promise(async (resolve, reject) => { + try { + const response = await this.http.request({ + url: `/license/features`, + method: 'get' + }); + + return resolve(response); + } catch (error) { + return reject(error); + } + }); + } + + + async setLicenses() { + try { + const response = await this.getLicenses(); + + let allowedLicenses: any[] = []; + Object.entries(response.data).forEach(([key, entry]: any) => { + if (entry.allowed) allowedLicenses.push(key); + }); + + localStorage.setItem('licenses', JSON.stringify(allowedLicenses)); + } catch {} + } + + hasLicense(license: string) { + const savedLicenses = localStorage.getItem('licenses'); + if (savedLicenses) { + const licenses = JSON.parse(savedLicenses); + const userHasLicense = licenses.includes(license); + + return userHasLicense; + } + + return false; + } +} diff --git a/web/ui/dashboard/src/assets/img/svg/page-locked.svg b/web/ui/dashboard/src/assets/img/svg/page-locked.svg new file mode 100644 index 0000000000..5a8679199c --- /dev/null +++ b/web/ui/dashboard/src/assets/img/svg/page-locked.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/web/ui/dashboard/src/index.html b/web/ui/dashboard/src/index.html index 26de11e99f..cd9cbe39b8 100644 --- a/web/ui/dashboard/src/index.html +++ b/web/ui/dashboard/src/index.html @@ -149,6 +149,12 @@ /> + + + + diff --git a/worker/task/process_broadcast_event_creation.go b/worker/task/process_broadcast_event_creation.go index 6fbb699633..8a9dcdea62 100644 --- a/worker/task/process_broadcast_event_creation.go +++ b/worker/task/process_broadcast_event_creation.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" - "gopkg.in/guregu/null.v4" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/frain-dev/convoy/queue/redis" "github.com/frain-dev/convoy" @@ -28,7 +30,7 @@ var ( defaultBroadcastDelay = 30 * time.Second ) -func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, subscriptionsTable memorystore.ITable) func(context.Context, *asynq.Task) error { +func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, licenser license.Licenser, subscriptionsTable memorystore.ITable) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) (err error) { var broadcastEvent models.BroadcastEvent @@ -77,7 +79,7 @@ func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, ev AcknowledgedAt: null.TimeFrom(time.Now()), } - subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, subscriptions, true) + subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, licenser, subscriptions, true) if err != nil { return &EndpointError{Err: fmt.Errorf("failed to match subscriptions using filter, err: %s", err.Error()), delay: defaultBroadcastDelay} } @@ -103,7 +105,7 @@ func ProcessBroadcastEventCreation(endpointRepo datastore.EndpointRepository, ev return nil } - err = writeEventDeliveriesToQueue(ctx, ss, event, project, eventDeliveryRepo, eventQueue, deviceRepo, endpointRepo) + err = writeEventDeliveriesToQueue(ctx, ss, event, project, eventDeliveryRepo, eventQueue, deviceRepo, endpointRepo, licenser) if err != nil { log.WithError(err).Error(ErrFailedToWriteToQueue) return &EndpointError{Err: fmt.Errorf("%s, err: %s", ErrFailedToWriteToQueue.Error(), err.Error()), delay: defaultBroadcastDelay} diff --git a/worker/task/process_broadcast_event_creation_test.go b/worker/task/process_broadcast_event_creation_test.go index 2f5b801b52..2e9c7ca77b 100644 --- a/worker/task/process_broadcast_event_creation_test.go +++ b/worker/task/process_broadcast_event_creation_test.go @@ -146,7 +146,7 @@ func TestProcessBroadcastEventCreation(t *testing.T) { fn := ProcessBroadcastEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, - args.deviceRepo, args.subTable) + args.deviceRepo, args.licenser, args.subTable) err = fn(context.Background(), task) if tt.wantErr { require.NotNil(t, err) diff --git a/worker/task/process_dynamic_event_creation.go b/worker/task/process_dynamic_event_creation.go index 7e80c2d8b3..d1ebdf4cb1 100644 --- a/worker/task/process_dynamic_event_creation.go +++ b/worker/task/process_dynamic_event_creation.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" - "gopkg.in/guregu/null.v4" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/frain-dev/convoy/pkg/msgpack" "github.com/google/uuid" @@ -23,7 +25,7 @@ import ( "github.com/oklog/ulid/v2" ) -func ProcessDynamicEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository) func(context.Context, *asynq.Task) error { +func ProcessDynamicEventCreation(endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, licenser license.Licenser) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var dynamicEvent models.DynamicEvent @@ -85,7 +87,7 @@ func ProcessDynamicEventCreation(endpointRepo datastore.EndpointRepository, even return writeEventDeliveriesToQueue( ctx, []datastore.Subscription{*s}, event, project, eventDeliveryRepo, - eventQueue, deviceRepo, endpointRepo, + eventQueue, deviceRepo, endpointRepo, licenser, ) } } diff --git a/worker/task/process_dynamic_event_creation_test.go b/worker/task/process_dynamic_event_creation_test.go index 193b5b0585..57447e8028 100644 --- a/worker/task/process_dynamic_event_creation_test.go +++ b/worker/task/process_dynamic_event_creation_test.go @@ -190,7 +190,7 @@ func TestProcessDynamicEventCreation(t *testing.T) { task := asynq.NewTask(string(convoy.EventProcessor), job.Payload, asynq.Queue(string(convoy.EventQueue)), asynq.ProcessIn(job.Delay)) - fn := ProcessDynamicEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo) + fn := ProcessDynamicEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo, args.licenser) err = fn(context.Background(), task) if tt.wantErr { require.NotNil(t, err) diff --git a/worker/task/process_event_creation.go b/worker/task/process_event_creation.go index a6bf845439..6d99b155a1 100644 --- a/worker/task/process_event_creation.go +++ b/worker/task/process_event_creation.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" - "gopkg.in/guregu/null.v4" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "gopkg.in/guregu/null.v4" + "github.com/frain-dev/convoy/pkg/flatten" "github.com/frain-dev/convoy" @@ -47,7 +49,7 @@ type CreateEvent struct { func ProcessEventCreation( endpointRepo datastore.EndpointRepository, eventRepo datastore.EventRepository, projectRepo datastore.ProjectRepository, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, - subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, + subRepo datastore.SubscriptionRepository, deviceRepo datastore.DeviceRepository, licenser license.Licenser, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var createEvent CreateEvent @@ -82,7 +84,7 @@ func ProcessEventCreation( event = createEvent.Event } - subscriptions, err := findSubscriptions(ctx, endpointRepo, subRepo, project, event, createEvent.CreateSubscription) + subscriptions, err := findSubscriptions(ctx, endpointRepo, subRepo, licenser, project, event, createEvent.CreateSubscription) if err != nil { return &EndpointError{Err: err, delay: defaultDelay} } @@ -112,12 +114,12 @@ func ProcessEventCreation( return writeEventDeliveriesToQueue( ctx, subscriptions, event, project, eventDeliveryRepo, - eventQueue, deviceRepo, endpointRepo, + eventQueue, deviceRepo, endpointRepo, licenser, ) } } -func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore.Subscription, event *datastore.Event, project *datastore.Project, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, deviceRepo datastore.DeviceRepository, endpointRepo datastore.EndpointRepository) error { +func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore.Subscription, event *datastore.Event, project *datastore.Project, eventDeliveryRepo datastore.EventDeliveryRepository, eventQueue queue.Queuer, deviceRepo datastore.DeviceRepository, endpointRepo datastore.EndpointRepository, licenser license.Licenser) error { ec := &EventDeliveryConfig{project: project} eventDeliveries := make([]*datastore.EventDelivery, 0) @@ -152,7 +154,7 @@ func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore. raw := event.Raw data := event.Data - if s.Function.Ptr() != nil && !util.IsStringEmpty(s.Function.String) { + if s.Function.Ptr() != nil && !util.IsStringEmpty(s.Function.String) && licenser.Transformations() { var payload map[string]interface{} err = json.Unmarshal(event.Data, &payload) if err != nil { @@ -252,7 +254,7 @@ func writeEventDeliveriesToQueue(ctx context.Context, subscriptions []datastore. } func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepository, - subRepo datastore.SubscriptionRepository, project *datastore.Project, event *datastore.Event, shouldCreateSubscription bool, + subRepo datastore.SubscriptionRepository, licenser license.Licenser, project *datastore.Project, event *datastore.Event, shouldCreateSubscription bool, ) ([]datastore.Subscription, error) { var subscriptions []datastore.Subscription var err error @@ -284,7 +286,7 @@ func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepos subs = matchSubscriptions(string(event.EventType), subs) - subs, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, subs, false) + subs, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, licenser, subs, false) if err != nil { return subscriptions, &EndpointError{Err: errors.New("error fetching subscriptions for event type"), delay: defaultDelay} } @@ -297,7 +299,7 @@ func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepos return nil, &EndpointError{Err: err, delay: defaultDelay} } - subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, subscriptions, false) + subscriptions, err = matchSubscriptionsUsingFilter(ctx, event, subRepo, licenser, subscriptions, false) if err != nil { log.WithError(err).Error("error find a matching subscription for this source") return subscriptions, &EndpointError{Err: errors.New("error find a matching subscription for this source"), delay: defaultDelay} @@ -307,7 +309,11 @@ func findSubscriptions(ctx context.Context, endpointRepo datastore.EndpointRepos return subscriptions, nil } -func matchSubscriptionsUsingFilter(ctx context.Context, e *datastore.Event, subRepo datastore.SubscriptionRepository, subscriptions []datastore.Subscription, soft bool) ([]datastore.Subscription, error) { +func matchSubscriptionsUsingFilter(ctx context.Context, e *datastore.Event, subRepo datastore.SubscriptionRepository, licenser license.Licenser, subscriptions []datastore.Subscription, soft bool) ([]datastore.Subscription, error) { + if !licenser.AdvancedSubscriptions() { + return subscriptions, nil + } + var matched []datastore.Subscription // payload is interface{} and not map[string]interface{} because diff --git a/worker/task/process_event_creation_test.go b/worker/task/process_event_creation_test.go index 5cb7480e68..f1a8931005 100644 --- a/worker/task/process_event_creation_test.go +++ b/worker/task/process_event_creation_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/internal/pkg/memorystore" @@ -31,6 +33,7 @@ type args struct { subRepo datastore.SubscriptionRepository deviceRepo datastore.DeviceRepository subTable memorystore.ITable + licenser license.Licenser } func provideArgs(ctrl *gomock.Controller) *args { @@ -56,6 +59,7 @@ func provideArgs(ctrl *gomock.Controller) *args { eventQueue: mockQueuer, subRepo: subRepo, subTable: subTable, + licenser: mocks.NewMockLicenser(ctrl), } } @@ -137,6 +141,86 @@ func TestProcessEventCreated(t *testing.T) { ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + + q, _ := args.eventQueue.(*mocks.MockQueuer) + q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(1).Return(nil) + }, + wantErr: false, + }, + + { + name: "should_skip_filter_comparing_for_license_check_failed", + event: &CreateEvent{ + Params: CreateEventTaskParams{ + UID: ulid.Make().String(), + EventType: "*", + SourceID: "source-id-1", + ProjectID: "project-id-1", + EndpointID: "endpoint-id-1", + Data: []byte(`{}`), + }, + }, + dbFn: func(args *args) { + project := &datastore.Project{ + UID: "project-id-1", + Type: datastore.OutgoingProject, + Config: &datastore.ProjectConfig{ + Strategy: &datastore.StrategyConfiguration{ + Type: datastore.LinearStrategyProvider, + Duration: 10, + RetryCount: 3, + }, + }, + } + + g, _ := args.projectRepo.(*mocks.MockProjectRepository) + g.EXPECT().FetchProjectByID(gomock.Any(), "project-id-1").Times(1).Return( + project, + nil, + ) + + a, _ := args.endpointRepo.(*mocks.MockEndpointRepository) + + endpoint := &datastore.Endpoint{UID: "endpoint-id-1"} + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()).Times(1).Return(endpoint, nil) + + s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) + subscriptions := []datastore.Subscription{ + { + UID: "456", + EndpointID: "endpoint-id-1", + Type: datastore.SubscriptionTypeAPI, + FilterConfig: &datastore.FilterConfiguration{ + EventTypes: []string{"*"}, + Filter: datastore.FilterSchema{ + Headers: nil, + Body: map[string]interface{}{"key": "value"}, + }, + }, + }, + } + + endpoint = &datastore.Endpoint{UID: "endpoint-id-1"} + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()).Times(1).Return(endpoint, nil) + + s.EXPECT().FindSubscriptionsByEndpointID(gomock.Any(), "project-id-1", "endpoint-id-1").Times(1).Return(subscriptions, nil) + + e, _ := args.eventRepo.(*mocks.MockEventRepository) + e.EXPECT().FindEventByID(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil, datastore.ErrEventNotFound) + e.EXPECT().CreateEvent(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + endpoint = &datastore.Endpoint{UID: "098", Url: "https://google.com", Status: datastore.ActiveEndpointStatus} + a.EXPECT().FindEndpointByID(gomock.Any(), "endpoint-id-1", gomock.Any()). + Times(1).Return(endpoint, nil) + + ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) + ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(false) + q, _ := args.eventQueue.(*mocks.MockQueuer) q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(1).Return(nil) }, @@ -284,6 +368,9 @@ func TestProcessEventCreated(t *testing.T) { ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + q, _ := args.eventQueue.(*mocks.MockQueuer) q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(2).Return(nil) }, @@ -365,6 +452,9 @@ func TestProcessEventCreated(t *testing.T) { }, nil, ) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + e, _ := args.eventRepo.(*mocks.MockEventRepository) e.EXPECT().FindEventByID(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil, datastore.ErrEventNotFound) e.EXPECT().CreateEvent(gomock.Any(), gomock.Any()).Times(1).Return(nil) @@ -442,6 +532,9 @@ func TestProcessEventCreated(t *testing.T) { ed, _ := args.eventDeliveryRepo.(*mocks.MockEventDeliveryRepository) ed.EXPECT().CreateEventDeliveries(gomock.Any(), gomock.Any()).Times(1).Return(nil) + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + q, _ := args.eventQueue.(*mocks.MockQueuer) q.EXPECT().Write(convoy.EventProcessor, convoy.EventQueue, gomock.Any()).Times(1).Return(nil) }, @@ -468,7 +561,7 @@ func TestProcessEventCreated(t *testing.T) { task := asynq.NewTask(string(convoy.EventProcessor), job.Payload, asynq.Queue(string(convoy.EventQueue)), asynq.ProcessIn(job.Delay)) - fn := ProcessEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo) + fn := ProcessEventCreation(args.endpointRepo, args.eventRepo, args.projectRepo, args.eventDeliveryRepo, args.eventQueue, args.subRepo, args.deviceRepo, args.licenser) err = fn(context.Background(), task) if tt.wantErr { require.NotNil(t, err) @@ -502,6 +595,47 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { dbFn: func(args *args) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) + }, + inputSubs: []datastore.Subscription{ + { + UID: "123", + FilterConfig: &datastore.FilterConfiguration{ + Filter: datastore.FilterSchema{ + Body: map[string]interface{}{"person.age": 10}, + }, + }, + }, + { + UID: "1234", + FilterConfig: &datastore.FilterConfiguration{ + Filter: datastore.FilterSchema{ + Body: map[string]interface{}{}, + }, + }, + }, + }, + wantSubs: []datastore.Subscription{ + { + UID: "123", + }, + { + UID: "1234", + }, + }, + }, + { + name: "Should skip filter for advanced subscriptions license check failed", + payload: map[string]interface{}{ + "person": map[string]interface{}{ + "age": 10, + }, + }, + dbFn: func(args *args) { + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(false) }, inputSubs: []datastore.Subscription{ { @@ -541,6 +675,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -573,6 +710,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -611,6 +751,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -649,6 +792,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -692,6 +838,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { dbFn: func(args *args) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(4).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -739,6 +888,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(4).Return(false, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -795,6 +947,9 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { s, _ := args.subRepo.(*mocks.MockSubscriptionRepository) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(false, nil) s.EXPECT().CompareFlattenedPayload(gomock.Any(), gomock.Any(), gomock.Any(), false).Times(2).Return(true, nil) + + licenser, _ := args.licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedSubscriptions().Times(1).Return(true) }, inputSubs: []datastore.Subscription{ { @@ -842,7 +997,7 @@ func TestMatchSubscriptionsUsingFilter(t *testing.T) { payload, err := json.Marshal(tt.payload) require.NoError(t, err) - subs, err := matchSubscriptionsUsingFilter(context.Background(), &datastore.Event{Data: payload}, args.subRepo, tt.inputSubs, false) + subs, err := matchSubscriptionsUsingFilter(context.Background(), &datastore.Event{Data: payload}, args.subRepo, args.licenser, tt.inputSubs, false) if tt.wantErr { require.NotNil(t, err) return diff --git a/worker/task/process_event_delivery.go b/worker/task/process_event_delivery.go index 3e5eff3913..530383166c 100644 --- a/worker/task/process_event_delivery.go +++ b/worker/task/process_event_delivery.go @@ -5,9 +5,12 @@ import ( "encoding/json" "errors" "fmt" - "github.com/frain-dev/convoy/internal/pkg/metrics" "time" + "github.com/frain-dev/convoy/internal/pkg/license" + + "github.com/frain-dev/convoy/internal/pkg/metrics" + "github.com/frain-dev/convoy/internal/pkg/limiter" "github.com/frain-dev/convoy/pkg/msgpack" @@ -28,7 +31,7 @@ import ( "github.com/hibiken/asynq" ) -func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, +func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDeliveryRepo datastore.EventDeliveryRepository, licenser license.Licenser, projectRepo datastore.ProjectRepository, q queue.Queuer, rateLimiter limiter.RateLimiter, dispatch *net.Dispatcher, attemptsRepo datastore.DeliveryAttemptsRepository, ) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) (err error) { @@ -160,7 +163,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive } var httpDuration time.Duration - if endpoint.HttpTimeout == 0 { + if endpoint.HttpTimeout == 0 || !licenser.AdvancedEndpointMgmt() { httpDuration = convoy.HTTP_TIMEOUT_IN_DURATION } else { httpDuration = time.Duration(endpoint.HttpTimeout) * time.Second @@ -193,7 +196,7 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive eventDelivery.LatencySeconds = time.Since(eventDelivery.GetLatencyStartTime()).Seconds() // register latency - mm := metrics.GetDPInstance() + mm := metrics.GetDPInstance(licenser) mm.RecordLatency(eventDelivery) } else { @@ -222,10 +225,12 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive log.WithError(err).Error("Failed to reactivate endpoint after successful retry") } - // send endpoint reactivation notification - err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, false, resp.Error, string(resp.Body), resp.StatusCode) - if err != nil { - log.FromContext(ctx).WithError(err).Error("failed to send notification") + if licenser.AdvancedEndpointMgmt() { + // send endpoint reactivation notification + err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, false, resp.Error, string(resp.Body), resp.StatusCode) + if err != nil { + log.FromContext(ctx).WithError(err).Error("failed to send notification") + } } } @@ -261,10 +266,12 @@ func ProcessEventDelivery(endpointRepo datastore.EndpointRepository, eventDelive log.WithError(err).Error("failed to deactivate endpoint after failed retry") } - // send endpoint deactivation notification - err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, true, resp.Error, string(resp.Body), resp.StatusCode) - if err != nil { - log.WithError(err).Error("failed to send notification") + if licenser.AdvancedEndpointMgmt() { + // send endpoint deactivation notification + err = notifications.SendEndpointNotification(ctx, endpoint, project, endpointStatus, q, true, resp.Error, string(resp.Body), resp.StatusCode) + if err != nil { + log.WithError(err).Error("failed to send notification") + } } } } diff --git a/worker/task/process_event_delivery_test.go b/worker/task/process_event_delivery_test.go index 4153ec0455..4e7eb94834 100644 --- a/worker/task/process_event_delivery_test.go +++ b/worker/task/process_event_delivery_test.go @@ -3,9 +3,12 @@ package task import ( "context" "encoding/json" + "testing" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/net" "github.com/stretchr/testify/require" - "testing" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth/realm_chain" @@ -26,7 +29,7 @@ func TestProcessEventDelivery(t *testing.T) { cfgPath string expectedError error msg *datastore.EventDelivery - dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository) + dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository, license.Licenser) nFn func() func() }{ { @@ -36,7 +39,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { m.EXPECT(). FindEventDeliveryByIDSlim(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.EventDelivery{ @@ -66,7 +69,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ RateLimit: 10, @@ -102,7 +105,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -184,7 +187,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ Secrets: []datastore.Secret{ @@ -250,6 +253,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -269,7 +275,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -335,6 +341,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -354,7 +363,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -420,6 +429,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -439,7 +451,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -507,6 +519,9 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -526,7 +541,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -591,6 +606,96 @@ func TestProcessEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) + }, + nFn: func() func() { + httpmock.Activate() + + httpmock.RegisterResponder("POST", "https://google.com", + httpmock.NewStringResponder(200, ``)) + + return func() { + httpmock.DeactivateAndReset() + } + }, + }, + { + name: "Manual retry - disable endpoint - success - advanced endpoint mgmt false", + cfgPath: "./testdata/Config/basic-convoy-disable-endpoint.json", + expectedError: nil, + msg: &datastore.EventDelivery{ + UID: "", + }, + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { + a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.Endpoint{ + ProjectID: "123", + Secrets: []datastore.Secret{ + {Value: "secret"}, + }, + RateLimit: 10, + RateLimitDuration: 60, + Status: datastore.ActiveEndpointStatus, + }, nil).Times(1) + + r.EXPECT().AllowWithDuration(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + + m.EXPECT(). + FindEventDeliveryByIDSlim(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.EventDelivery{ + Status: datastore.ScheduledEventStatus, + Metadata: &datastore.Metadata{ + Data: []byte(`{"event": "invoice.completed"}`), + Raw: `{"event": "invoice.completed"}`, + NumTrials: 4, + RetryLimit: 3, + IntervalSeconds: 20, + }, + }, nil).Times(1) + + m.EXPECT(). + UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + o.EXPECT(). + FetchProjectByID(gomock.Any(), gomock.Any()). + Return(&datastore.Project{ + LogoURL: "", + Config: &datastore.ProjectConfig{ + Signature: &datastore.SignatureConfiguration{ + Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Versions: []datastore.SignatureVersion{ + { + UID: "abc", + Hash: "SHA256", + Encoding: datastore.HexEncoding, + }, + }, + }, + SSL: &datastore.DefaultSSLConfig, + Strategy: &datastore.StrategyConfiguration{ + Type: datastore.LinearStrategyProvider, + Duration: 60, + RetryCount: 1, + }, + RateLimit: &datastore.DefaultRateLimitConfig, + DisableEndpoint: true, + }, + }, nil).Times(1) + + a.EXPECT().UpdateEndpointStatus(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + d.EXPECT().CreateDeliveryAttempt(gomock.Any(), gomock.Any()).Times(1) + + m.EXPECT(). + UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(false) }, nFn: func() func() { httpmock.Activate() @@ -610,7 +715,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -680,6 +785,9 @@ func TestProcessEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -699,7 +807,7 @@ func TestProcessEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -770,6 +878,9 @@ func TestProcessEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedEndpointMgmt().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -799,6 +910,9 @@ func TestProcessEventDelivery(t *testing.T) { q := mocks.NewMockQueuer(ctrl) rateLimiter := mocks.NewMockRateLimiter(ctrl) attemptsRepo := mocks.NewMockDeliveryAttemptsRepository(ctrl) + licenser := mocks.NewMockLicenser(ctrl) + + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) err := config.LoadConfig(tc.cfgPath) if err != nil { @@ -821,13 +935,13 @@ func TestProcessEventDelivery(t *testing.T) { } if tc.dbFn != nil { - tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo) + tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo, licenser) } - dispatcher, err := net.NewDispatcher("", false) + dispatcher, err := net.NewDispatcher("", licenser, false) require.NoError(t, err) - processFn := ProcessEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo) + processFn := ProcessEventDelivery(endpointRepo, msgRepo, licenser, projectRepo, q, rateLimiter, dispatcher, attemptsRepo) payload := EventDelivery{ EventDeliveryID: tc.msg.UID, diff --git a/worker/task/process_meta_event.go b/worker/task/process_meta_event.go index 788f537f12..57263bab71 100644 --- a/worker/task/process_meta_event.go +++ b/worker/task/process_meta_event.go @@ -28,7 +28,7 @@ type MetaEvent struct { ProjectID string } -func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo datastore.MetaEventRepository) func(context.Context, *asynq.Task) error { +func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo datastore.MetaEventRepository, dispatch *net.Dispatcher) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var data MetaEvent @@ -73,7 +73,7 @@ func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo dat delayDuration := retrystrategies.NewRetryStrategyFromMetadata(*metaEvent.Metadata).NextDuration(metaEvent.Metadata.NumTrials) - resp, err := sendUrlRequest(ctx, project, metaEvent) + resp, err := sendUrlRequest(ctx, project, metaEvent, dispatch) metaEvent.Metadata.NumTrials++ if resp != nil { @@ -120,19 +120,12 @@ func ProcessMetaEvent(projectRepo datastore.ProjectRepository, metaEventRepo dat } } -func sendUrlRequest(ctx context.Context, project *datastore.Project, metaEvent *datastore.MetaEvent) (*net.Response, error) { +func sendUrlRequest(ctx context.Context, project *datastore.Project, metaEvent *datastore.MetaEvent, dispatch *net.Dispatcher) (*net.Response, error) { cfg, err := config.Get() if err != nil { return nil, err } - httpDuration := convoy.HTTP_TIMEOUT_IN_DURATION - dispatch, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, project.Config.SSL.EnforceSecureEndpoints) - if err != nil { - log.WithError(err).Error("error occurred while creating http client") - return nil, err - } - sig := &signature.Signature{ Payload: json.RawMessage(metaEvent.Metadata.Raw), Schemes: []signature.Scheme{ @@ -152,6 +145,7 @@ func sendUrlRequest(ctx context.Context, project *datastore.Project, metaEvent * url := project.Config.MetaEvent.URL + httpDuration := convoy.HTTP_TIMEOUT_IN_DURATION resp, err := dispatch.SendRequest(ctx, url, string(convoy.HttpPost), sig.Payload, "X-Convoy-Signature", header, int64(cfg.MaxResponseSize), httpheader.HTTPHeader{}, dedup.GenerateChecksum(metaEvent.UID), httpDuration) if err != nil { return nil, err diff --git a/worker/task/process_meta_event_test.go b/worker/task/process_meta_event_test.go index 9dc6391dbb..c35c0111ec 100644 --- a/worker/task/process_meta_event_test.go +++ b/worker/task/process_meta_event_test.go @@ -6,6 +6,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + + "github.com/frain-dev/convoy/net" + "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" @@ -129,8 +133,13 @@ func TestProcessMetaEvent(t *testing.T) { metaEventRepo := mocks.NewMockMetaEventRepository(ctrl) projectRepo := mocks.NewMockProjectRepository(ctrl) + licenser := mocks.NewMockLicenser(ctrl) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) + + dispatcher, err := net.NewDispatcher("", licenser, false) + require.NoError(t, err) - err := config.LoadConfig(tc.cfgPath) + err = config.LoadConfig(tc.cfgPath) if err != nil { t.Errorf("failed to load config file: %v", err) } @@ -144,7 +153,7 @@ func TestProcessMetaEvent(t *testing.T) { tc.dbFn(metaEventRepo, projectRepo) } - processFn := ProcessMetaEvent(projectRepo, metaEventRepo) + processFn := ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher) payload := MetaEvent{ MetaEventID: tc.msg.MetaEventID, ProjectID: tc.msg.ProjectID, diff --git a/worker/task/process_retry_event_delivery_test.go b/worker/task/process_retry_event_delivery_test.go index 55f2570fbb..3903e663f5 100644 --- a/worker/task/process_retry_event_delivery_test.go +++ b/worker/task/process_retry_event_delivery_test.go @@ -4,9 +4,12 @@ import ( "context" "encoding/json" "fmt" + "testing" + + "github.com/frain-dev/convoy/internal/pkg/license" + "github.com/frain-dev/convoy/net" "github.com/stretchr/testify/require" - "testing" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/auth/realm_chain" @@ -27,7 +30,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { cfgPath string expectedError error msg *datastore.EventDelivery - dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository) + dbFn func(*mocks.MockEndpointRepository, *mocks.MockProjectRepository, *mocks.MockEventDeliveryRepository, *mocks.MockQueuer, *mocks.MockRateLimiter, *mocks.MockDeliveryAttemptsRepository, license.Licenser) nFn func() func() }{ { @@ -37,7 +40,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { m.EXPECT(). FindEventDeliveryByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.EventDelivery{ @@ -58,6 +61,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { project := &datastore.Project{UID: "project-id-1"} o.EXPECT().FetchProjectByID(gomock.Any(), "project-id-1").Times(1).Return(project, nil) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, }, { @@ -67,7 +73,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ RateLimit: 10, @@ -94,6 +100,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, }, { @@ -103,7 +112,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -157,6 +166,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { d.EXPECT().CreateDeliveryAttempt(gomock.Any(), gomock.Any()).Times(1) + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) + m.EXPECT(). UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) @@ -183,7 +195,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ Secrets: []datastore.Secret{ @@ -220,7 +232,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -249,6 +261,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -268,7 +283,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -305,7 +320,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -316,7 +331,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { }, SSL: &datastore.DefaultSSLConfig, Strategy: &datastore.StrategyConfiguration{ - Type: datastore.StrategyProvider("default"), + Type: "default", Duration: 60, RetryCount: 1, }, @@ -334,6 +349,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -353,7 +371,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -390,7 +408,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -419,6 +437,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -438,7 +459,97 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { + a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.Endpoint{ + ProjectID: "123", + Url: "https://google.com?source=giphy", + Secrets: []datastore.Secret{ + {Value: "secret"}, + }, + RateLimit: 10, + RateLimitDuration: 60, + Status: datastore.ActiveEndpointStatus, + }, nil) + + r.EXPECT().AllowWithDuration(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + + m.EXPECT(). + FindEventDeliveryByID(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&datastore.EventDelivery{ + Status: datastore.ScheduledEventStatus, + URLQueryParams: "name=ref&category=food", + Metadata: &datastore.Metadata{ + Data: []byte(`{"event": "invoice.completed"}`), + Raw: `{"event": "invoice.completed"}`, + NumTrials: 4, + RetryLimit: 3, + IntervalSeconds: 20, + }, + }, nil).Times(1) + + a.EXPECT(). + UpdateEndpointStatus(gomock.Any(), gomock.Any(), gomock.Any(), datastore.InactiveEndpointStatus). + Return(nil).Times(1) + + o.EXPECT(). + FetchProjectByID(gomock.Any(), gomock.Any()). + Return(&datastore.Project{ + LogoURL: "", + Config: &datastore.ProjectConfig{ + Signature: &datastore.SignatureConfiguration{ + Header: "X-Convoy-Signature", + Versions: []datastore.SignatureVersion{ + { + UID: "abc", + Hash: "SHA256", + Encoding: datastore.HexEncoding, + }, + }, + }, + SSL: &datastore.DefaultSSLConfig, + Strategy: &datastore.StrategyConfiguration{ + Type: datastore.LinearStrategyProvider, + Duration: 60, + RetryCount: 1, + }, + RateLimit: &datastore.DefaultRateLimitConfig, + DisableEndpoint: true, + }, + }, nil).Times(1) + + d.EXPECT().CreateDeliveryAttempt(gomock.Any(), gomock.Any()).Times(1) + + m.EXPECT(). + UpdateStatusOfEventDelivery(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + m.EXPECT(). + UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) + }, + nFn: func() func() { + httpmock.Activate() + + httpmock.RegisterResponder("POST", "https://google.com?category=food&name=ref&source=giphy", + httpmock.NewStringResponder(200, ``)) + + return func() { + httpmock.DeactivateAndReset() + } + }, + }, + { + name: "Manual retry - disable endpoint - success - skip proxy", + cfgPath: "./testdata/Config/basic-convoy.json", + expectedError: nil, + msg: &datastore.EventDelivery{ + UID: "", + }, + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -477,7 +588,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -506,6 +617,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(false) }, nFn: func() func() { httpmock.Activate() @@ -525,7 +639,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -562,7 +676,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -590,6 +704,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { m.EXPECT(). UpdateEventDeliveryMetadata(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -609,7 +726,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -647,7 +764,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { LogoURL: "", Config: &datastore.ProjectConfig{ Signature: &datastore.SignatureConfiguration{ - Header: config.SignatureHeaderProvider("X-Convoy-Signature"), + Header: "X-Convoy-Signature", Versions: []datastore.SignatureVersion{ { UID: "abc", @@ -679,6 +796,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -698,7 +818,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { msg: &datastore.EventDelivery{ UID: "", }, - dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository) { + dbFn: func(a *mocks.MockEndpointRepository, o *mocks.MockProjectRepository, m *mocks.MockEventDeliveryRepository, q *mocks.MockQueuer, r *mocks.MockRateLimiter, d *mocks.MockDeliveryAttemptsRepository, l license.Licenser) { a.EXPECT().FindEndpointByID(gomock.Any(), gomock.Any(), gomock.Any()). Return(&datastore.Endpoint{ ProjectID: "123", @@ -769,6 +889,9 @@ func TestProcessRetryEventDelivery(t *testing.T) { q.EXPECT(). Write(convoy.NotificationProcessor, convoy.DefaultQueue, gomock.Any()). Return(nil).Times(1) + + licenser, _ := l.(*mocks.MockLicenser) + licenser.EXPECT().UseForwardProxy().Times(1).Return(true) }, nFn: func() func() { httpmock.Activate() @@ -798,6 +921,7 @@ func TestProcessRetryEventDelivery(t *testing.T) { q := mocks.NewMockQueuer(ctrl) rateLimiter := mocks.NewMockRateLimiter(ctrl) attemptsRepo := mocks.NewMockDeliveryAttemptsRepository(ctrl) + licenser := mocks.NewMockLicenser(ctrl) err := config.LoadConfig(tc.cfgPath) if err != nil { @@ -820,10 +944,10 @@ func TestProcessRetryEventDelivery(t *testing.T) { } if tc.dbFn != nil { - tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo) + tc.dbFn(endpointRepo, projectRepo, msgRepo, q, rateLimiter, attemptsRepo, licenser) } - dispatcher, err := net.NewDispatcher("", false) + dispatcher, err := net.NewDispatcher("", licenser, false) require.NoError(t, err) processFn := ProcessRetryEventDelivery(endpointRepo, msgRepo, projectRepo, q, rateLimiter, dispatcher, attemptsRepo) From 89c13c2f205bdb3ddd045b98d27217f40229034a Mon Sep 17 00:00:00 2001 From: Pelumi Muyiwa-Oni Date: Mon, 2 Sep 2024 01:40:16 +0100 Subject: [PATCH 11/16] add convoy logo to portal links (#2132) * add convoy logo to portal links * feat: modify link to open a new tab and added utm tracking --------- Co-authored-by: Subomi Oluwalana --- docs/docs.go | 2 +- .../src/app/portal/portal.component.html | 50 +++++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 1ba132971a..339862aaed 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1,4 +1,4 @@ -// Package docs Code generated by swaggo/swag at 2024-08-30 12:08:22.812424 +0100 BST m=+2.397263876. DO NOT EDIT +// Package docs Code generated by swaggo/swag at 2024-09-01 19:32:01.187081 -0500 CDT m=+1.863614585. DO NOT EDIT package docs import "github.com/swaggo/swag" diff --git a/web/ui/dashboard/src/app/portal/portal.component.html b/web/ui/dashboard/src/app/portal/portal.component.html index 539ebb87ca..4fcabc375e 100644 --- a/web/ui/dashboard/src/app/portal/portal.component.html +++ b/web/ui/dashboard/src/app/portal/portal.component.html @@ -1,23 +1,31 @@ -
    - +
    + +
    + +
    + + convoy logo +
    - From 4d296d38813a3d700acf89097968798d70e84c94 Mon Sep 17 00:00:00 2001 From: Daniel Oluojomu Date: Mon, 2 Sep 2024 16:17:57 +0100 Subject: [PATCH 12/16] Gate event search & consumer pool tuning (#2134) * fix: gate event search & consumer pool tuning * update license gating for search policy (#2133) * add notifications gating --------- Co-authored-by: Pelumi Muyiwa-Oni --- api/api.go | 2 - api/handlers/event.go | 5 + api/models/project.go | 1 + cmd/hooks/hooks.go | 8 ++ cmd/worker/worker.go | 2 +- docs/docs.go | 2 +- internal/pkg/license/keygen/feature.go | 2 + internal/pkg/license/keygen/keygen.go | 16 ++++ internal/pkg/license/keygen/keygen_test.go | 10 ++ internal/pkg/license/license.go | 2 + internal/pkg/license/noop/noop.go | 11 +++ mocks/license.go | 28 ++++++ services/create_endpoint.go | 3 + services/create_endpoint_test.go | 6 +- services/project_service.go | 8 ++ services/project_service_test.go | 93 ++++++++++++++++++- services/update_endpoint.go | 5 +- .../create-endpoint.component.html | 12 ++- .../create-project-component.component.html | 34 ++++--- .../create-project-component.component.ts | 2 +- .../create-project-component.module.ts | 4 +- .../event-delivery-filter.component.html | 16 +--- .../event-delivery-filter.component.ts | 3 +- 23 files changed, 234 insertions(+), 41 deletions(-) diff --git a/api/api.go b/api/api.go index e868e60d84..70c3606ae5 100644 --- a/api/api.go +++ b/api/api.go @@ -174,7 +174,6 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { eventRouter.With(handler.RequireEnabledProject()).Post("/fanout", handler.CreateEndpointFanoutEvent) eventRouter.With(handler.RequireEnabledProject()).Post("/broadcast", handler.CreateBroadcastEvent) eventRouter.With(handler.RequireEnabledProject()).Post("/dynamic", handler.CreateDynamicEvent) - eventRouter.With(handler.RequireEnabledProject()).With(middleware.Pagination).Get("/", handler.GetEventsPaged) eventRouter.With(handler.RequireEnabledProject()).Post("/batchreplay", handler.BatchReplayEvents) eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { @@ -343,7 +342,6 @@ func (a *ApplicationHandler) BuildControlPlaneRoutes() *chi.Mux { eventRouter.With(handler.RequireEnabledProject()).Post("/fanout", handler.CreateEndpointFanoutEvent) eventRouter.With(handler.RequireEnabledProject()).Post("/broadcast", handler.CreateBroadcastEvent) eventRouter.With(handler.RequireEnabledProject()).Post("/dynamic", handler.CreateDynamicEvent) - eventRouter.With(handler.RequireEnabledProject()).With(middleware.Pagination).Get("/", handler.GetEventsPaged) eventRouter.With(handler.RequireEnabledProject()).Post("/batchreplay", handler.BatchReplayEvents) eventRouter.Route("/{eventID}", func(eventSubRouter chi.Router) { diff --git a/api/handlers/event.go b/api/handlers/event.go index 54da990508..6ab7645305 100644 --- a/api/handlers/event.go +++ b/api/handlers/event.go @@ -477,6 +477,11 @@ func (h *Handler) GetEventsPaged(w http.ResponseWriter, r *http.Request) { } data.Filter.Project = project + + if !h.A.Licenser.AdvancedWebhookFiltering() { + data.Filter.Query = "" // event payload search not allowed + } + eventsPaged, paginationData, err := postgres.NewEventRepo(h.A.DB, h.A.Cache).LoadEventsPaged(r.Context(), project.UID, data.Filter) if err != nil { log.FromContext(r.Context()).WithError(err).Error("failed to fetch events") diff --git a/api/models/project.go b/api/models/project.go index f2f7f86b84..656c1aeaeb 100644 --- a/api/models/project.go +++ b/api/models/project.go @@ -88,6 +88,7 @@ func (pc *ProjectConfig) Transform() *datastore.ProjectConfig { AddEventIDTraceHeaders: pc.AddEventIDTraceHeaders, MultipleEndpointSubscriptions: pc.MultipleEndpointSubscriptions, SSL: pc.SSL.transform(), + SearchPolicy: pc.SearchPolicy, RateLimit: pc.RateLimit.Transform(), Strategy: pc.Strategy.transform(), Signature: pc.Signature.transform(), diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index b2d6f970c1..ab6c8bb54e 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -200,6 +200,14 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [ return err } + if !app.Licenser.ConsumerPoolTuning() { + cfg.ConsumerPoolSize = config.DefaultConfiguration.ConsumerPoolSize + } + + if err = config.Override(&cfg); err != nil { + return err + } + // update config singleton with the instance id if _, ok := skipConfigLoadCmd[cmd.Use]; !ok { configRepo := postgres.NewConfigRepo(app.DB) diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go index bdbf1071ec..bef3f59df0 100644 --- a/cmd/worker/worker.go +++ b/cmd/worker/worker.go @@ -314,7 +314,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte if err != nil { return nil } - if fflag.CanAccessFeature(fflag2.FullTextSearch) { + if fflag.CanAccessFeature(fflag2.FullTextSearch) && a.Licenser.AdvancedWebhookFiltering() { consumer.RegisterHandlers(convoy.TokenizeSearch, task.GeneralTokenizerHandler(projectRepo, eventRepo, jobRepo, rd), nil) consumer.RegisterHandlers(convoy.TokenizeSearchForProject, task.TokenizerHandler(eventRepo, jobRepo), nil) } diff --git a/docs/docs.go b/docs/docs.go index 339862aaed..0a4007ee3f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1,4 +1,4 @@ -// Package docs Code generated by swaggo/swag at 2024-09-01 19:32:01.187081 -0500 CDT m=+1.863614585. DO NOT EDIT +// Package docs Code generated by swaggo/swag at 2024-09-02 15:03:52.730374 +0100 BST m=+2.026310793. DO NOT EDIT package docs import "github.com/swaggo/swag" diff --git a/internal/pkg/license/keygen/feature.go b/internal/pkg/license/keygen/feature.go index 4e13d73c66..678ea5bc1b 100644 --- a/internal/pkg/license/keygen/feature.go +++ b/internal/pkg/license/keygen/feature.go @@ -22,6 +22,8 @@ const ( AsynqMonitoring Feature = "ASYNQ_MONITORING" SynchronousWebhooks Feature = "SYNCHRONOUS_WEBHOOKS" PortalLinks Feature = "PORTAL_LINKS" + ConsumerPoolTuning Feature = "CONSUMER_POOL_TUNING" + AdvancedWebhookFiltering Feature = "ADVANCED_WEBHOOK_FILTERING" ) const ( diff --git a/internal/pkg/license/keygen/keygen.go b/internal/pkg/license/keygen/keygen.go index 62bdd07b77..981583472f 100644 --- a/internal/pkg/license/keygen/keygen.go +++ b/internal/pkg/license/keygen/keygen.go @@ -427,6 +427,22 @@ func (k *Licenser) PortalLinks() bool { return ok } +func (k *Licenser) ConsumerPoolTuning() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[ConsumerPoolTuning] + return ok +} + +func (k *Licenser) AdvancedWebhookFiltering() bool { + if checkExpiry(k.license) != nil { + return false + } + _, ok := k.featureList[AdvancedWebhookFiltering] + return ok +} + func (k *Licenser) FeatureListJSON(ctx context.Context) (json.RawMessage, error) { // only these guys have dynamic limits for now for f := range k.featureList { diff --git a/internal/pkg/license/keygen/keygen_test.go b/internal/pkg/license/keygen/keygen_test.go index 9d3f8861b6..a14738f807 100644 --- a/internal/pkg/license/keygen/keygen_test.go +++ b/internal/pkg/license/keygen/keygen_test.go @@ -86,6 +86,16 @@ func TestKeygenLicenserBoolMethods(t *testing.T) { k = Licenser{featureList: map[Feature]*Properties{PortalLinks: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} require.False(t, k.PortalLinks()) + k = Licenser{featureList: map[Feature]*Properties{ConsumerPoolTuning: {}}, license: &keygen.License{}} + require.True(t, k.ConsumerPoolTuning()) + k = Licenser{featureList: map[Feature]*Properties{ConsumerPoolTuning: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.ConsumerPoolTuning()) + + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookFiltering: {}}, license: &keygen.License{}} + require.True(t, k.AdvancedWebhookFiltering()) + k = Licenser{featureList: map[Feature]*Properties{AdvancedWebhookFiltering: {}}, license: &keygen.License{Expiry: timePtr(time.Now().Add(-400000 * time.Hour))}} + require.False(t, k.AdvancedWebhookFiltering()) + k = Licenser{enabledProjects: map[string]bool{ "12345": true, }} diff --git a/internal/pkg/license/license.go b/internal/pkg/license/license.go index ed2ae65969..faf6a693a1 100644 --- a/internal/pkg/license/license.go +++ b/internal/pkg/license/license.go @@ -19,6 +19,8 @@ type Licenser interface { Transformations() bool AsynqMonitoring() bool PortalLinks() bool + ConsumerPoolTuning() bool + AdvancedWebhookFiltering() bool // need more fleshing out AdvancedRetentionPolicy() bool diff --git a/internal/pkg/license/noop/noop.go b/internal/pkg/license/noop/noop.go index c1fcf897e4..6cc613e271 100644 --- a/internal/pkg/license/noop/noop.go +++ b/internal/pkg/license/noop/noop.go @@ -78,12 +78,23 @@ func (Licenser) MutualTLS() bool { func (Licenser) SynchronousWebhooks() bool { return true } + func (Licenser) RemoveEnabledProject(_ string) {} + func (Licenser) ProjectEnabled(_ string) bool { return true } + func (Licenser) AddEnabledProject(_ string) {} +func (Licenser) ConsumerPoolTuning() bool { + return true +} + +func (Licenser) AdvancedWebhookFiltering() bool { + return true +} + func (Licenser) PortalLinks() bool { return true } diff --git a/mocks/license.go b/mocks/license.go index 5ff9840ca8..29d8166aaa 100644 --- a/mocks/license.go +++ b/mocks/license.go @@ -108,6 +108,20 @@ func (mr *MockLicenserMockRecorder) AdvancedSubscriptions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedSubscriptions", reflect.TypeOf((*MockLicenser)(nil).AdvancedSubscriptions)) } +// AdvancedWebhookFiltering mocks base method. +func (m *MockLicenser) AdvancedWebhookFiltering() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvancedWebhookFiltering") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AdvancedWebhookFiltering indicates an expected call of AdvancedWebhookFiltering. +func (mr *MockLicenserMockRecorder) AdvancedWebhookFiltering() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvancedWebhookFiltering", reflect.TypeOf((*MockLicenser)(nil).AdvancedWebhookFiltering)) +} + // AsynqMonitoring mocks base method. func (m *MockLicenser) AsynqMonitoring() bool { m.ctrl.T.Helper() @@ -136,6 +150,20 @@ func (mr *MockLicenserMockRecorder) CanExportPrometheusMetrics() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanExportPrometheusMetrics", reflect.TypeOf((*MockLicenser)(nil).CanExportPrometheusMetrics)) } +// ConsumerPoolTuning mocks base method. +func (m *MockLicenser) ConsumerPoolTuning() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConsumerPoolTuning") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ConsumerPoolTuning indicates an expected call of ConsumerPoolTuning. +func (mr *MockLicenserMockRecorder) ConsumerPoolTuning() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsumerPoolTuning", reflect.TypeOf((*MockLicenser)(nil).ConsumerPoolTuning)) +} + // CreateOrg mocks base method. func (m *MockLicenser) CreateOrg(ctx context.Context) (bool, error) { m.ctrl.T.Helper() diff --git a/services/create_endpoint.go b/services/create_endpoint.go index ea71ef746a..36f0564628 100644 --- a/services/create_endpoint.go +++ b/services/create_endpoint.go @@ -76,6 +76,9 @@ func (a *CreateEndpointService) Run(ctx context.Context) (*datastore.Endpoint, e if !a.Licenser.AdvancedEndpointMgmt() { // switch to default timeout endpoint.HttpTimeout = convoy.HTTP_TIMEOUT + + endpoint.SupportEmail = "" + endpoint.SlackWebhookURL = "" } if util.IsStringEmpty(endpoint.AppID) { diff --git a/services/create_endpoint_test.go b/services/create_endpoint_test.go index aced45986d..de0b7352b4 100644 --- a/services/create_endpoint_test.go +++ b/services/create_endpoint_test.go @@ -94,7 +94,7 @@ func TestCreateEndpointService_Run(t *testing.T) { wantErr: false, }, { - name: "should_default_http_timeout_endpoint_for_license_check", + name: "should_default_http_timeout_endpoint_for_license_check_and_remove_slack_url_support_email", args: args{ ctx: ctx, e: models.CreateEndpoint{ @@ -126,8 +126,8 @@ func TestCreateEndpointService_Run(t *testing.T) { }, wantEndpoint: &datastore.Endpoint{ Name: "endpoint", - SupportEmail: "endpoint@test.com", - SlackWebhookURL: "https://google.com", + SupportEmail: "", + SlackWebhookURL: "", ProjectID: project.UID, Secrets: []datastore.Secret{ {Value: "1234"}, diff --git a/services/project_service.go b/services/project_service.go index ca2b6c9a53..0370498baa 100644 --- a/services/project_service.go +++ b/services/project_service.go @@ -88,6 +88,10 @@ func (ps *ProjectService) CreateProject(ctx context.Context, newProject *models. } } + if !ps.Licenser.AdvancedWebhookFiltering() { + projectConfig.SearchPolicy = "" + } + project := &datastore.Project{ UID: ulid.Make().String(), Name: projectName, @@ -177,6 +181,10 @@ func (ps *ProjectService) UpdateProject(ctx context.Context, project *datastore. project.LogoURL = update.LogoURL } + if !ps.Licenser.AdvancedWebhookFiltering() { + project.Config.SearchPolicy = "" + } + err := ps.projectRepo.UpdateProject(ctx, project) if err != nil { log.FromContext(ctx).WithError(err).Error("failed to to update project") diff --git a/services/project_service_test.go b/services/project_service_test.go index deda3d055b..f76ed1df11 100644 --- a/services/project_service_test.go +++ b/services/project_service_test.go @@ -56,6 +56,7 @@ func TestProjectService_CreateProject(t *testing.T) { Signature: &models.SignatureConfiguration{ Header: "X-Convoy-Signature", }, + SearchPolicy: "300h", Strategy: &models.StrategyConfiguration{ Type: "linear", Duration: 20, @@ -88,6 +89,7 @@ func TestProjectService_CreateProject(t *testing.T) { licenser, _ := gs.Licenser.(*mocks.MockLicenser) licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project", @@ -103,7 +105,8 @@ func TestProjectService_CreateProject(t *testing.T) { Duration: 20, RetryCount: 4, }, - SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: true}, + SearchPolicy: "300h", + SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: true}, RateLimit: &datastore.RateLimitConfiguration{ Count: 1000, Duration: 60, @@ -126,6 +129,7 @@ func TestProjectService_CreateProject(t *testing.T) { Signature: &models.SignatureConfiguration{ Header: "X-Convoy-Signature", }, + SearchPolicy: "300h", Strategy: &models.StrategyConfiguration{ Type: "linear", Duration: 20, @@ -159,6 +163,7 @@ func TestProjectService_CreateProject(t *testing.T) { licenser, _ := gs.Licenser.(*mocks.MockLicenser) licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project", @@ -169,7 +174,8 @@ func TestProjectService_CreateProject(t *testing.T) { Signature: &datastore.SignatureConfiguration{ Header: "X-Convoy-Signature", }, - SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: false}, + SearchPolicy: "300h", + SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: false}, Strategy: &datastore.StrategyConfiguration{ Type: "linear", Duration: 20, @@ -215,6 +221,7 @@ func TestProjectService_CreateProject(t *testing.T) { licenser, _ := gs.Licenser.(*mocks.MockLicenser) licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project_1", @@ -272,6 +279,7 @@ func TestProjectService_CreateProject(t *testing.T) { licenser, _ := gs.Licenser.(*mocks.MockLicenser) licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantProject: &datastore.Project{ Name: "test_project", @@ -299,6 +307,79 @@ func TestProjectService_CreateProject(t *testing.T) { }, wantErr: false, }, + { + name: "should_remove_search_policy_for_license_check", + args: args{ + ctx: ctx, + newProject: &models.CreateProject{ + Name: "test_project", + Type: "outgoing", + LogoURL: "https://google.com", + Config: &models.ProjectConfig{ + Signature: &models.SignatureConfiguration{ + Header: "X-Convoy-Signature", + }, + SearchPolicy: "300h", + Strategy: &models.StrategyConfiguration{ + Type: "linear", + Duration: 20, + RetryCount: 4, + }, + RateLimit: &models.RateLimitConfiguration{ + Count: 1000, + Duration: 60, + }, + ReplayAttacks: true, + }, + }, + org: &datastore.Organisation{UID: "1234"}, + member: &datastore.OrganisationMember{ + UID: "abc", + OrganisationID: "1234", + Role: auth.Role{Type: auth.RoleSuperUser}, + }, + }, + dbFn: func(gs *ProjectService) { + a, _ := gs.projectRepo.(*mocks.MockProjectRepository) + a.EXPECT().CreateProject(gomock.Any(), gomock.Any()). + Times(1).Return(nil) + + a.EXPECT().FetchProjectByID(gomock.Any(), gomock.Any()).Times(1).Return(&datastore.Project{UID: "abc", OrganisationID: "1234"}, nil) + + apiKeyRepo, _ := gs.apiKeyRepo.(*mocks.MockAPIKeyRepository) + apiKeyRepo.EXPECT().CreateAPIKey(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AddEnabledProject(gomock.Any()) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(false) + }, + wantProject: &datastore.Project{ + Name: "test_project", + Type: "outgoing", + LogoURL: "https://google.com", + OrganisationID: "1234", + Config: &datastore.ProjectConfig{ + Signature: &datastore.SignatureConfiguration{ + Header: "X-Convoy-Signature", + }, + Strategy: &datastore.StrategyConfiguration{ + Type: "linear", + Duration: 20, + RetryCount: 4, + }, + SearchPolicy: "", + SSL: &datastore.SSLConfiguration{EnforceSecureEndpoints: true}, + RateLimit: &datastore.RateLimitConfiguration{ + Count: 1000, + Duration: 60, + }, + // RetentionPolicy: &datastore.DefaultRetentionPolicy, + ReplayAttacks: true, + }, + }, + wantErr: false, + }, { name: "should_fail_to_create_project", args: args{ @@ -329,6 +410,7 @@ func TestProjectService_CreateProject(t *testing.T) { licenser, _ := gs.Licenser.(*mocks.MockLicenser) licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantErr: true, wantErrCode: http.StatusBadRequest, @@ -403,6 +485,7 @@ func TestProjectService_CreateProject(t *testing.T) { licenser, _ := gs.Licenser.(*mocks.MockLicenser) licenser.EXPECT().CreateProject(gomock.Any()).Times(1).Return(true, nil) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) }, wantErr: true, wantErrCode: http.StatusBadRequest, @@ -563,6 +646,9 @@ func TestProjectService_UpdateProject(t *testing.T) { }, }, dbFn: func(gs *ProjectService) { + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) + a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().UpdateProject(gomock.Any(), gomock.Any()).Times(1).Return(nil) }, @@ -590,6 +676,9 @@ func TestProjectService_UpdateProject(t *testing.T) { }, }, dbFn: func(gs *ProjectService) { + licenser, _ := gs.Licenser.(*mocks.MockLicenser) + licenser.EXPECT().AdvancedWebhookFiltering().Times(1).Return(true) + a, _ := gs.projectRepo.(*mocks.MockProjectRepository) a.EXPECT().UpdateProject(gomock.Any(), gomock.Any()).Times(1).Return(errors.New("failed")) }, diff --git a/services/update_endpoint.go b/services/update_endpoint.go index a0381cbad9..431b6dbafa 100644 --- a/services/update_endpoint.go +++ b/services/update_endpoint.go @@ -5,6 +5,7 @@ import ( "time" "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/internal/pkg/license" "github.com/frain-dev/convoy/pkg/log" @@ -62,11 +63,11 @@ func (a *UpdateEndpointService) updateEndpoint(endpoint *datastore.Endpoint, e m endpoint.Name = *e.Name - if e.SupportEmail != nil { + if e.SupportEmail != nil && a.Licenser.AdvancedEndpointMgmt() { endpoint.SupportEmail = *e.SupportEmail } - if e.SlackWebhookURL != nil { + if e.SlackWebhookURL != nil && a.Licenser.AdvancedEndpointMgmt() { endpoint.SlackWebhookURL = *e.SlackWebhookURL } diff --git a/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html b/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html index cac80ee125..b84c0b3714 100644 --- a/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html +++ b/web/ui/dashboard/src/app/private/components/create-endpoint/create-endpoint.component.html @@ -81,7 +81,15 @@
    -

    Alert Configuration

    +
    +

    Alert Configuration

    +
    + + + + Business +
    +
    -
    +
    diff --git a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.html b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.html index 6c8e86ae39..01b0d924de 100644 --- a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.html +++ b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.html @@ -147,11 +147,19 @@

    Project type

    -
    -

    - Search Period - This will trigger search re-tokenization and only events within this period will be available for search. -

    +
    +
    +

    + Search Period + This will trigger search re-tokenization and only events within this period will be available for search. +

    +
    + + + + Business +
    +
    - -
    - -
    hour(s)
    -
    - Enter search policy value -     + + +
    + +
    hour(s)
    +
    + Enter search policy value +     +
    diff --git a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts index 20d71e464b..b043f7feea 100644 --- a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts +++ b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.component.ts @@ -103,7 +103,7 @@ export class CreateProjectComponent implements OnInit { private privateService: PrivateService, public router: Router, private route: ActivatedRoute, - private licenseService: LicensesService + public licenseService: LicensesService ) {} async ngOnInit() { diff --git a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts index 1792d827b5..91000b9ca9 100644 --- a/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts +++ b/web/ui/dashboard/src/app/private/components/create-project-component/create-project-component.module.ts @@ -16,6 +16,7 @@ import { TokenModalComponent } from '../token-modal/token-modal.component'; import { PermissionDirective } from '../permission/permission.directive'; import { NotificationComponent } from 'src/app/components/notification/notification.component'; import { ConfigButtonComponent } from '../config-button/config-button.component'; +import { TagComponent } from 'src/app/components/tag/tag.component'; @NgModule({ declarations: [CreateProjectComponent], @@ -46,7 +47,8 @@ import { ConfigButtonComponent } from '../config-button/config-button.component' PermissionDirective, DialogDirective, NotificationComponent, - ConfigButtonComponent + ConfigButtonComponent, + TagComponent ], exports: [CreateProjectComponent] }) diff --git a/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html b/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html index 838c51f78f..19f377cb3b 100644 --- a/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html +++ b/web/ui/dashboard/src/app/private/components/event-delivery-filter/event-delivery-filter.component.html @@ -1,7 +1,7 @@
    -
    +
    search icon @@ -21,17 +21,7 @@
    Date
    - {{ queryParams.startDate | date: 'dd/MM/yy, h:mm a' }} - {{ queryParams.endDate | date: 'dd/MM/yy, h:mm a' }} + {{ queryParams.startDate | date : 'dd/MM/yy, h:mm a' }} - {{ queryParams.endDate | date : 'dd/MM/yy, h:mm a' }}