Skip to content

refactor: migration tooling #11225

refactor: migration tooling

refactor: migration tooling #11225

Workflow file for this run

name: CI
on:
push:
branches: [main]
pull_request:
permissions:
contents: read
env:
DAGGER_VERSION: 0.18.14
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
cache-rebuild:
name: Rebuild GitHub Actions Caches
runs-on: depot-ubuntu-latest-4
# To test e2e feel free to comment this out for a branch, then a branch specific cache will be created
if: github.event_name == 'push'
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Build nix environment
run: |
nix flake check --impure
nix develop --impure .#ci
- name: Save Nix store cache
uses: nix-community/cache-nix-action/save@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
save: "true"
- name: Populate go caches - go mod download
run: nix develop --impure .#ci -c go mod download
- name: Save go caches - go mod
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
build:
name: Build
runs-on: depot-ubuntu-latest-8
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Restore Nix store
uses: nix-community/cache-nix-action/restore@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
restore-prefixes-first-match: |
${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-
${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }}
${{ runner.os }}-openmeter-nix-build-main-
${{ runner.os }}-openmeter-nix-build-
- name: Restore go.mod cache if exists
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
# Prefer to restore the branch cache over the main cache
restore-keys: |
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-
${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-main
- name: Validate Nix flake
run: nix flake check --impure
- name: Build components
run: nix develop --impure .#ci -c make -j 3 build
- name: Validate commit messages
run: |
nix develop --impure .#ci -c pre-commit run -a
nix develop --impure .#ci -c pre-commit run --hook-stage manual
generators:
name: Code Generators
runs-on: depot-ubuntu-latest-8
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Restore Nix store
uses: nix-community/cache-nix-action/restore@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
restore-prefixes-first-match: |
${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-
${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }}
${{ runner.os }}-openmeter-nix-build-main-
${{ runner.os }}-openmeter-nix-build-
- name: Restore go.mod cache if exists
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
# Prefer to restore the branch cache over the main cache
restore-keys: |
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-
${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-main
- name: Ensure code generators are run
run: |
nix develop --impure .#ci -c make generate-all
# does not detect new files
if [ -n "$(git diff --exit-code)" ]; then
git diff
echo "Code generators have not been run, please run 'make generate-all' and commit the changes"
exit 1
fi
# detect files not committed
if [ -n "$(git status --porcelain)" ]; then
git status --porcelain
echo "Code generators have not been run, please run 'make generate-all' and commit the changes"
exit 1
fi
test:
name: Test
runs-on: depot-ubuntu-latest-8
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
# Let's start docker-compose early so that by the time we have the nix env set up it's already running
- name: Start docker-compose dependencies
run: |
docker compose up postgres svix redis -d
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Restore Nix store
uses: nix-community/cache-nix-action/restore@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
restore-prefixes-first-match: |
${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-
${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }}
${{ runner.os }}-openmeter-nix-build-main-
${{ runner.os }}-openmeter-nix-build-
- name: Upsert Nix store
run: nix develop --impure .#ci
# This shaves off 5s or so
- name: Restore go.mod cache if exists
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
# Prefer to restore the branch cache over the main cache
restore-keys: |
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-
${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-main
- name: Wait for dependencies to be ready
run: |
./tools/wait-for-compose.sh postgres svix redis
- name: Run tests
env:
SVIX_HOST: localhost
# Dev JWT secret, non-sensitive
SVIX_JWT_SECRET: DUMMY_JWT_SECRET
# count=1 is needed to force retest
run: |
nix develop --impure .#ci -c make test-nocache
- name: Stop docker-compose dependencies
if: always()
run: |
docker compose down -v
migrations:
name: Migration Checks
runs-on: depot-ubuntu-latest-8
steps:
# Required as a workaround for Dagger to properly detect Git metadata
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0 # Needed to compare against base branch
persist-credentials: false
- name: Run pipeline
uses: dagger/dagger-for-github@d913e70051faf3b907d4dd96ef1161083c88c644 # v8.2.0
with:
verb: call
args: migrate check
cloud-token: ${{ secrets.DAGGER_CLOUD_NO_CACHE_TOKEN }}
version: ${{ env.DAGGER_VERSION }}
- name: Export Dagger Engine logs
id: export-dagger-engine-logs
run: docker logs $(docker container list --all --filter 'name=^dagger-engine-*' --format '{{.Names}}') > engine.stdout.log 2> engine.stderr.log
if: always()
continue-on-error: true
- name: Upload Dagger Engine logs as artifact
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
if: always() && steps.export-dagger-engine-logs.outcome == 'success'
with:
name: "[${{ github.job }}] Dagger Engine logs"
path: |
engine.stdout.log
engine.stderr.log
retention-days: 14
lint:
name: Lint
runs-on: depot-ubuntu-latest-8
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Restore Nix store
uses: nix-community/cache-nix-action/restore@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
restore-prefixes-first-match: |
${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-
${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }}
${{ runner.os }}-openmeter-nix-build-main-
${{ runner.os }}-openmeter-nix-build-
- name: Upsert Nix store
run: nix develop --impure .#ci
# This shaves off 5s or so
- name: Restore go.mod cache if exists
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
# Prefer to restore the branch cache over the main cache
restore-keys: |
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-
${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-main
- name: Get main fork point
id: base-commit
run: |
git fetch origin main
echo "base candidate:"
git log -1 --reverse --boundary HEAD ^origin/main
BASE_COMMIT=$(git log -1 --reverse --boundary --format=%h HEAD ^origin/main)
if [ -z "$BASE_COMMIT" ]; then
BASE_COMMIT=$(git log -1 --format=%h)
fi
echo "sha=${BASE_COMMIT}" >> $GITHUB_OUTPUT
echo "sha=${BASE_COMMIT}"
- name: Lint cache
uses: actions/[email protected]
with:
path: .devenv/golangci-lint-cache
key: ${{ runner.os }}-openmeter-golangci-lint-cache-${{ steps.base-commit.outputs.sha }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
restore-keys: |
${{ runner.os }}-openmeter-golangci-lint-cache-${{ steps.base-commit.outputs.sha }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-golangci-lint-cache-${{ steps.base-commit.outputs.sha }}-
${{ runner.os }}-openmeter-golangci-lint-cache-
- name: Run linters - go
env:
GOLANGCI_LINT_CACHE: ${{ github.workspace }}/.devenv/golangci-lint-cache
run: |
nix develop --impure .#ci -c make lint-go
- name: Run linters - api spec
run: |
nix develop --impure .#ci -c make lint-api-spec
- name: Run linters - openapi
run: |
nix develop --impure .#ci -c make lint-openapi
- name: Run linters - helm
run: |
nix develop --impure .#ci -c make lint-helm
artifacts:
name: Artifacts
uses: ./.github/workflows/artifacts.yaml
with:
publish: ${{ github.event_name == 'push' }}
permissions:
contents: read
packages: write
id-token: write
security-events: write
dependency-review:
name: Dependency review
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Dependency Review
uses: actions/dependency-review-action@40c09b7dc99638e5ddb0bfd91c1673effc064d8a # v4.8.1
fossa-scan:
name: FOSSA Scan
runs-on: ubuntu-latest
if: github.event_name == 'push'
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Run FOSSA Scan
uses: fossas/fossa-action@3ebcea1862c6ffbd5cf1b4d0bd6b3fe7bd6f2cac # v1.7.0
with:
api-key: ${{secrets.FOSSA_API_KEY}}
quickstart:
name: Quickstart
runs-on: depot-ubuntu-latest-8
needs:
- artifacts
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Create override files for quickstart
env:
DEPOT_IMAGE_URL: ${{ needs.artifacts.outputs.container-image-url-depot }}
run: |
cat > quickstart/docker-compose.override.yaml <<EOF
services:
openmeter:
image: $DEPOT_IMAGE_URL
sink-worker:
image: $DEPOT_IMAGE_URL
balance-worker:
image: $DEPOT_IMAGE_URL
notification-service:
image: $DEPOT_IMAGE_URL
billing-worker:
image: $DEPOT_IMAGE_URL
openmeter-jobs:
image: $DEPOT_IMAGE_URL
EOF
cat quickstart/docker-compose.override.yaml
- name: Launch Docker Compose
run: docker compose -f docker-compose.yaml -f docker-compose.override.yaml up -d
working-directory: quickstart
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Restore Nix store
uses: nix-community/cache-nix-action/restore@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
restore-prefixes-first-match: |
${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-
${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }}
${{ runner.os }}-openmeter-nix-build-main-
${{ runner.os }}-openmeter-nix-build-
- name: Restore go.mod cache if exists
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
# Prefer to restore the branch cache over the main cache
restore-keys: |
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-
${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-main
- name: Check container health
run: docker inspect --format "{{json .State.Health }}" $(docker container list --all --filter 'name=^*-openmeter-*' --format '{{.Names}}')
if: always()
continue-on-error: true
- name: Wait for worker to become ready
run: |
curl --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:40000/healthz
docker ps
- name: Run tests
env:
OPENMETER_ADDRESS: http://localhost:48888
run: |
nix develop --impure .#ci -c go test -v -count=1 ./quickstart/
- name: Cleanup Docker Compose
run: docker compose -f docker-compose.yaml -f docker-compose.override.yaml down -v
working-directory: quickstart
if: always()
e2e:
name: E2E
runs-on: depot-ubuntu-latest-8
# Note: This check is running against the image that is going to be pushed.
needs:
- artifacts
steps:
- name: Checkout repository
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
persist-credentials: false
- name: Create override files for e2e
env:
DEPOT_IMAGE_URL: ${{ needs.artifacts.outputs.container-image-url-depot }}
run: |
cat > e2e/docker-compose.override.yaml <<EOF
services:
openmeter:
image: $DEPOT_IMAGE_URL
sink-worker:
image: $DEPOT_IMAGE_URL
EOF
cat e2e/docker-compose.override.yaml
- name: Launch Docker Compose infra
run: docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml up -d
working-directory: e2e
- name: Set up Nix
uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
nix_conf: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
keep-env-derivations = true
keep-outputs = true
- name: Restore Nix store
uses: nix-community/cache-nix-action/restore@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
with:
primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }}
restore-prefixes-first-match: |
${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-
${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }}
${{ runner.os }}-openmeter-nix-build-main-
${{ runner.os }}-openmeter-nix-build-
- name: Restore go.mod cache if exists
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .devenv/state/go
key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
# Prefer to restore the branch cache over the main cache
restore-keys: |
${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-
${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }}
${{ runner.os }}-openmeter-go-modules-main
- name: Check container health
run: docker inspect --format "{{json .State.Health }}" $(docker container list --all --filter 'name=^*-openmeter-*' --format '{{.Names}}')
if: always()
continue-on-error: true
- name: Wait for worker to become ready
run: |
curl --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30000/healthz
docker ps
- name: Run tests
env:
OPENMETER_ADDRESS: http://localhost:38888
TZ: UTC
run: |
nix develop --impure .#ci -c go test -v -count=1 -timeout 3m ./e2e/
- name: Cleanup Docker Compose
run: docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml down -v
working-directory: e2e
if: always()
- name: Upload Openmeter logs as artifact
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
if: always()
with:
name: "[${{ github.job }}] Openmeter logs"
path: |
e2e/logs/openmeter/openmeter.log
e2e/logs/sink-worker/openmeter.log
retention-days: 14