Make changelog polling signals pluggable

.devcontainer/Containerfile

@@ -0,0 +1,21 @@
+FROM docker.io/golang:1-bookworm
+
+RUN apt-get update && apt-get install --yes \
+    curl wget gpg \
+    make git vim less \
+    sudo \
+    bash-completion man \
+    gnuplot
+
+# Create a non-root user so linux users can run the container as the current
+# OS user. See https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
+# and /~https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-reference.md#container-creation
+# for more information.
+ARG USERNAME=dev
+ARG USER_UID=1010
+ARG USER_GID=1010
+
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME

.devcontainer/devcontainer.json

@@ -0,0 +1,38 @@
+{
+    "name": "sqlite-cdc",
+    "build": {
+        "dockerfile": "Containerfile",
+        "context": ".."
+    },
+    "remoteUser": "dev",
+    "updateRemoteUserUID": true,
+    "containerEnv": {},
+    "mounts": [],
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "telemetry.telemetryLevel": "off",
+                "telemetry.enableTelemetry": false,
+                "files.insertFinalNewline": true,
+                "files.trimTrailingWhitespace": true,
+                "rewrap.wrappingColumn": 80,
+                "go.formatTool": "goimports",
+                "go.lintTool": "golangci-lint",
+                "terminal.integrated.profiles.linux": {
+                    "bash": {
+                        "path": "/usr/bin/bash"
+                    }
+                },
+                "terminal.integrated.defaultProfile.linux": "bash"
+            },
+            "extensions": [
+                "golang.go",
+                "streetsidesoftware.code-spell-checker",
+                "stkb.rewrap",
+                "ms-vscode.makefile-tools",
+                "redhat.vscode-yaml",
+                "ms-vscode.cpptools-extension-pack"
+            ]
+        }
+    }
+}

.github/workflows/go.yml → .github/workflows/pr.yml

@@ -1,16 +1,18 @@
 # This workflow will build a golang project
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
 
-name: Go
+name: Pull Request Checks
 
 on:
   push:
-    branches: [ "main" ]
+    branches:
+      - main
   pull_request:
-    branches: [ "main" ]
+    branches:
+      - "**"
+  workflow_dispatch: {}
 
 jobs:
-
   build:
     runs-on: ubuntu-latest
     steps:
@@ -20,8 +22,14 @@ jobs:
         cache-dependency-path: |
              go.sum
              internal/tools/go.sum
-
+    - uses: actions/cache@v4
+      id: tools
+      with:
+        path: |
+          .bin
+        key: ${{ runner.os }}-${{ hashFiles('internal/tools/go.sum') }}
     - name: Tools
+      if: steps.tools.outputs.cache-hit != 'true'
       run: make tools
     - name: Lint
       run: make test/lint

.github/workflows/tag.yaml

@@ -0,0 +1,40 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: Tag And Release
+
+on:
+  push:
+    tags:
+      - "**"
+  workflow_dispatch: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        cache-dependency-path: |
+             go.sum
+             internal/tools/go.sum
+    - uses: actions/cache@v4
+      id: tools
+      with:
+        path: |
+          .bin
+        key: ${{ runner.os }}-${{ hashFiles('internal/tools/go.sum') }}
+    - name: Tools
+      if: steps.tools.outputs.cache-hit != 'true'
+      run: make tools
+    - name: Lint
+      run: make test/lint
+    - name: Unit Test
+      run: make test/unit
+    - name: Generate builds
+      run: make BUILD_MODE=tag build
+    - name: Publish release
+      run: make release
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.golangci.yaml

@@ -3,16 +3,12 @@ run:
   issues-exit-code: 2
   tests: true
   build-tags: []
-  skip-dirs:
-    - \.bin$
-    - \.coverage$
-  skip-dirs-use-default: true
-  skip-files: []
   modules-download-mode: readonly
   allow-parallel-runners: false
 
 output:
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
   print-issued-lines: true
   print-linter-name: true
   uniq-by-line: true
@@ -47,6 +43,10 @@ issues:
         - gosec
   exclude-use-default: true
   exclude-case-sensitive: false
+  exclude-dirs:
+    - \.bin$
+    - \.coverage$
+  exclude-dirs-use-default: true
   include: []
   max-issues-per-linter: 50
   max-same-issues: 3

.goreleaser.yaml

@@ -0,0 +1,52 @@
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+
+version: 2
+
+builds:
+  - id: sqlite-cdc
+    binary: sqlite-cdc
+    main: ./cmd/sqlite-cdc/main.go
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+    flags:
+      - -trimpath
+    ldflags:
+      - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
+  - id: sqlite-cdc-setup
+    binary: sqlite-cdc-setup
+    main: ./cmd/sqlite-cdc-setup/main.go
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+    flags:
+      - -trimpath
+    ldflags:
+      - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of `uname`.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        format: zip
+
+checksum:
+  split: true
+
+changelog:
+  sort: asc

Makefile

@@ -8,7 +8,12 @@ GOTOOLS = $(shell grep '_' $(TOOLS_DIR)/tools.go | sed 's/[[:space:]]*_//g' | se
 BUILD_DIR = $(PROJECT_PATH)/.build
 TOOLS_DIR = $(PROJECT_PATH)/internal/tools
 TOOLS_FILE = $(TOOLS_DIR)/tools.go
+DIST_DIR = $(PROJECT_PATH)/dist
 BIN_DIR = $(PROJECT_PATH)/.bin
+BENCH_DIR = $(BUILD_DIR)/.bench
+IMAGES_DIR = $(PROJECT_PATH)/images
+IMAGES_PLOTS = $(wildcard $(IMAGES_DIR)/*.plot)
+IMAGES_PLOTS_RENDERED = $(subst .plot,.svg,$(IMAGES_PLOTS))
 COVER_DIR = $(BUILD_DIR)/.coverage
 COVERAGE_UNIT = $(COVER_DIR)/unit.out
 COVERAGE_UNIT_INTERCHANGE = $(COVERAGE_UNIT:.out=.interchange)
@@ -21,6 +26,11 @@ COVERAGE_COMBINED_XML = $(COVERAGE_COMBINED:.out=.xml)
 GOIMPORT_LOCAL = github.com/kevinconway
 GOLANGCILINT_CONFIG = $(PROJECT_PATH)/.golangci.yaml
 GOCMD = GOFLAGS=$(GOFLAGS) go
+BUILD_MODE = local
+BUILD_FLAGS = --clean
+ifneq ($(BUILD_MODE),tag)
+	BUILD_FLAGS = --clean --snapshot
+endif
 
 #######
 # https://stackoverflow.com/a/10858332
@@ -32,6 +42,12 @@ __check_defined = \
       $(error Undefined $1$(if $2, ($2))))
 #######
 
+build: | $(BIN_DIR) $(DIST_DIR)
+	@ $(BIN_DIR)/goreleaser build $(BUILD_FLAGS)
+
+release: | $(BIN_DIR) $(DIST_DIR)
+	@ $(BIN_DIR)/goreleaser release --clean
+
 test: test/lint test/unit test/coverage
 
 test/lint: | $(BIN_DIR)
@@ -44,14 +60,74 @@ test/unit: $(COVERAGE_UNIT) | $(BIN_DIR)
 test/coverage: $(COVER_DIR) $(COVERAGE_UNIT) $(COVERAGE_UNIT_INTERCHANGE) $(COVERATE_UNIT_HTML) $(COVERAGE_UNIT_XML) $(COVERAGE_COMBINED) $(COVERAGE_COMBINED_INTERCHANGE) $(COVERAGE_COMBINED_HTML) $(COVERAGE_COMBINED_XML) | $(BIN_DIR)
 	@ $(GOCMD) tool cover -func $(COVERAGE_COMBINED)
 
+BENCH_TRIGGER_SERIAL_INSERT = $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialInserts.txt
+BENCH_TRIGGER_SERIAL_INSERT_REPORT = $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialInserts-cmp.csv
+BENCH_TRIGGER_SERIAL_UPDATE = $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialUpdates.txt
+BENCH_TRIGGER_SERIAL_UPDATE_REPORT = $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialUpdates-cmp.csv
+BENCH_TRIGGER_SERIAL_DELETE = $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialDeletes.txt
+BENCH_TRIGGER_SERIAL_DELETE_REPORT = $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialDeletes-cmp.csv
+BENCH_TRIGGER_SERIAL_WIDE_INSERT = $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialInserts.txt
+BENCH_TRIGGER_SERIAL_WIDE_INSERT_REPORT = $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialInserts-cmp.csv
+BENCH_TRIGGER_SERIAL_WIDE_UPDATE = $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialUpdates.txt
+BENCH_TRIGGER_SERIAL_WIDE_UPDATE_REPORT = $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialUpdates-cmp.csv
+BENCH_TRIGGER_SERIAL_WIDE_DELETE = $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialDeletes.txt
+BENCH_TRIGGER_SERIAL_WIDE_DELETE_REPORT = $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialDeletes-cmp.csv
+BENCH_BLOB = $(BENCH_DIR)/BenchmarkBlobEncoding.txt
+BENCH_BLOB_REPORT = $(BENCH_DIR)/BenchmarkBlobEncoding-cmp.csv
+
+benchmarks: benchmarks/simple benchmarks/wide benchmarks/blob
+benchmarks/simple: $(BENCH_TRIGGER_SERIAL_INSERT) $(BENCH_TRIGGER_SERIAL_UPDATE) $(BENCH_TRIGGER_SERIAL_DELETE)
+benchmarks/wide: $(BENCH_TRIGGER_SERIAL_WIDE_INSERT) $(BENCH_TRIGGER_SERIAL_WIDE_UPDATE) $(BENCH_TRIGGER_SERIAL_WIDE_DELETE)
+benchmarks/blob: $(BENCH_BLOB)
+
+benchmark-reports: benchmark-reports/simple benchmark-reports/wide benchmark-reports/blob
+benchmark-reports/simple: $(BENCH_TRIGGER_SERIAL_INSERT_REPORT) $(BENCH_TRIGGER_SERIAL_UPDATE_REPORT) $(BENCH_TRIGGER_SERIAL_DELETE_REPORT)
+benchmark-reports/wide: $(BENCH_TRIGGER_SERIAL_WIDE_INSERT_REPORT) $(BENCH_TRIGGER_SERIAL_WIDE_UPDATE_REPORT) $(BENCH_TRIGGER_SERIAL_WIDE_DELETE_REPORT)
+benchmark-reports/blob: $(BENCH_BLOB_REPORT)
+
+$(BENCH_TRIGGER_SERIAL_INSERT): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkTriggerLatencySimpleTableSerialInserts*' -count=20 > $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialInserts.txt
+$(BENCH_TRIGGER_SERIAL_UPDATE): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkTriggerLatencySimpleTableSerialUpdates*' -count=20 > $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialUpdates.txt
+$(BENCH_TRIGGER_SERIAL_DELETE): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkTriggerLatencySimpleTableSerialDeletes*' -count=20 > $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialDeletes.txt
+$(BENCH_TRIGGER_SERIAL_WIDE_INSERT): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkTriggerLatencyWideTableSerialInserts*' -count=20 > $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialInserts.txt
+$(BENCH_TRIGGER_SERIAL_WIDE_UPDATE): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkTriggerLatencyWideTableSerialUpdates*' -count=20 > $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialUpdates.txt
+$(BENCH_TRIGGER_SERIAL_WIDE_DELETE): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkTriggerLatencyWideTableSerialDeletes*' -count=20 > $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialDeletes.txt
+$(BENCH_BLOB): | $(BIN_DIR) $(BENCH_DIR)
+	@ $(GOCMD) test -timeout 0 -run='^$$' -bench='^BenchmarkBlobEncoding*' -count=20 > $(BENCH_DIR)/BenchmarkBlobEncoding.txt
+
+$(BENCH_TRIGGER_SERIAL_INSERT_REPORT): $(BENCH_TRIGGER_SERIAL) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /triggers -row /columns $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialInserts.txt > $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialInserts-cmp.csv
+$(BENCH_TRIGGER_SERIAL_UPDATE_REPORT): $(BENCH_TRIGGER_SERIAL) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /triggers -row /columns $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialUpdates.txt > $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialUpdates-cmp.csv
+$(BENCH_TRIGGER_SERIAL_DELETE_REPORT): $(BENCH_TRIGGER_SERIAL) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /triggers -row /columns $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialDeletes.txt > $(BENCH_DIR)/BenchmarkTriggerLatencySimpleTableSerialDeletes-cmp.csv
+$(BENCH_TRIGGER_SERIAL_WIDE_INSERT_REPORT): $(BENCH_TRIGGER_SERIAL) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /triggers -row /columns $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialInserts.txt > $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialInserts-cmp.csv
+$(BENCH_TRIGGER_SERIAL_WIDE_UPDATE_REPORT): $(BENCH_TRIGGER_SERIAL) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /triggers -row /columns $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialUpdates.txt > $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialUpdates-cmp.csv
+$(BENCH_TRIGGER_SERIAL_WIDE_DELETE_REPORT): $(BENCH_TRIGGER_SERIAL) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /triggers -row /columns $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialDeletes.txt > $(BENCH_DIR)/BenchmarkTriggerLatencyWideTableSerialDeletes-cmp.csv
+$(BENCH_BLOB_REPORT): $(BENCH_BLOB) | $(BIN_DIR)
+	@ $(BIN_DIR)/benchstat -format csv -col /size $(BENCH_DIR)/BenchmarkBlobEncoding.txt > $(BENCH_DIR)/BenchmarkBlobEncoding-cmp.csv
+
+images: $(IMAGES_PLOTS_RENDERED)
+
+$(IMAGES_DIR)/%.svg: $(IMAGES_DIR)/%.plot
+	@ gnuplot $^ > $@
+
 tools: | $(BIN_DIR)
 	@ cd $(TOOLS_DIR) && GOBIN=$(BIN_DIR) $(GOCMD) install $(GOTOOLS)
 tools/update:
 	@ cd $(TOOLS_DIR) && GOBIN=$(BIN_DIR) $(GOCMD) get -u
 	@ cd $(TOOLS_DIR) && GOBIN=$(BIN_DIR) $(GOCMD) mod tidy
 
 $(BIN_DIR):
-	mkdir -p $(BIN_DIR)
+	@ mkdir -p $(BIN_DIR)
 
 generate:
 	@ go generate ./...
@@ -62,9 +138,12 @@ fmt: | $(BIN_DIR)
 		-local $(GOIMPORT_LOCAL) \
 		$(shell find . -type f -name '*.go' -not -path "./vendor/*")
 
-clean: clean/test clean/tools
+clean: clean/test clean/tools clean/build
+clean/build:
 	@:$(call check_defined,BUILD_DIR)
 	@ rm -rf "$(BUILD_DIR)"
+	@:$(call check_defined,DIST_DIR)
+	@ rm -rf "$(DIST_DIR)"
 clean/test:
 	@:$(call check_defined,COVER_DIR)
 	@ rm -rf "$(COVER_DIR)"
@@ -74,7 +153,7 @@ clean/tools:
 
 
 $(COVERAGE_UNIT): $(shell find . -type f -name '*.go' -not -path "./vendor/*") | $(COVER_DIR)
-	$(GOCMD) test \
+	@ $(GOCMD) test \
 		-v \
 		-cover \
 		-race \
@@ -99,5 +178,11 @@ $(COVERAGE_COMBINED):
 	@ GOFLAGS="$(GOFLAGS)" \
  	$(BIN_DIR)/gocovmerge $(COVER_DIR)/*.out > $(COVERAGE_COMBINED)
 
-$(COVER_DIR): 
+$(COVER_DIR): | $(BUILD_DIR)
 	@ mkdir -p $(COVER_DIR)
+
+$(BENCH_DIR): | $(BUILD_DIR)
+	@ mkdir -p $(BENCH_DIR)
+
+$(BUILD_DIR):
+	@ mkdir -p $(BUILD_DIR)