From 783d4c35cfd8086d0ca977d1465dfaccad8dbfc6 Mon Sep 17 00:00:00 2001 From: Tutus Development Date: Sat, 27 Dec 2025 15:45:20 +0000 Subject: [PATCH] Rebrand to Tutus - update license, workflows, and dependencies --- .gitattributes | 8 +- .github/dependabot.yml | 22 +- .github/workflows/benchmark-pr.yaml | 18 +- .github/workflows/benchmark-releases.yaml | 26 +- .github/workflows/benchmark-template.yaml | 114 +-- .github/workflows/failpoint_test.yaml | 40 +- .github/workflows/gh-workflow-approve.yaml | 84 +- .github/workflows/robustness_nightly.yaml | 34 +- .github/workflows/robustness_template.yaml | 100 +-- .github/workflows/robustness_test.yaml | 32 +- .github/workflows/stale.yaml | 38 +- .github/workflows/tests-template.yml | 110 +-- .github/workflows/tests_amd64.yaml | 52 +- .github/workflows/tests_arm64.yaml | 52 +- .github/workflows/tests_windows.yml | 114 +-- .gitignore | 24 +- .go-version | 2 +- .golangci.yaml | 68 +- CHANGELOG/CHANGELOG-1.3.md | 180 ++-- CHANGELOG/CHANGELOG-1.4.md | 152 ++-- LICENSE | 40 +- Makefile | 216 ++--- OWNERS | 20 +- cmd/bbolt/OWNERS | 24 +- cmd/bbolt/README.md | 906 ++++++++++----------- code-of-conduct.md | 6 +- go.sum | 48 +- scripts/compare_benchmarks.sh | 140 ++-- scripts/fix.sh | 26 +- 29 files changed, 1348 insertions(+), 1348 deletions(-) diff --git a/.gitattributes b/.gitattributes index a681ce3..4b4eb2b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ -# ensure that line endings for Windows builds are properly formatted -# see https://github.com/golangci/golangci-lint-action?tab=readme-ov-file#how-to-use -# at "Multiple OS Example" section -*.go text eol=lf +# ensure that line endings for Windows builds are properly formatted +# see https://github.com/golangci/golangci-lint-action?tab=readme-ov-file#how-to-use +# at "Multiple OS Example" section +*.go text eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml index aafb8a2..14153ea 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,11 @@ -version: 2 -updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - - - package-ecosystem: gomod - directory: / - schedule: - interval: weekly +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/benchmark-pr.yaml b/.github/workflows/benchmark-pr.yaml index 95de955..1945e49 100644 --- a/.github/workflows/benchmark-pr.yaml +++ b/.github/workflows/benchmark-pr.yaml @@ -1,9 +1,9 @@ ---- -name: Benchmarks on PRs (AMD64) -permissions: read-all -on: [pull_request] -jobs: - amd64: - uses: ./.github/workflows/benchmark-template.yaml - with: - benchGitRef: ${{ github.event.pull_request.base.sha }} +--- +name: Benchmarks on PRs (AMD64) +permissions: read-all +on: [pull_request] +jobs: + amd64: + uses: ./.github/workflows/benchmark-template.yaml + with: + benchGitRef: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/benchmark-releases.yaml b/.github/workflows/benchmark-releases.yaml index 6cc1c1f..b4bb5ff 100644 --- a/.github/workflows/benchmark-releases.yaml +++ b/.github/workflows/benchmark-releases.yaml @@ -1,13 +1,13 @@ ---- -name: Nightly Benchmarks against last release (AMD64) -permissions: read-all -on: - schedule: - - cron: '10 5 * * *' # runs every day at 05:10 UTC - # workflow_dispatch enables manual testing of this job by maintainers - workflow_dispatch: -jobs: - amd64: - uses: ./.github/workflows/benchmark-template.yaml - with: - benchGitRef: release-1.3 +--- +name: Nightly Benchmarks against last release (AMD64) +permissions: read-all +on: + schedule: + - cron: '10 5 * * *' # runs every day at 05:10 UTC + # workflow_dispatch enables manual testing of this job by maintainers + workflow_dispatch: +jobs: + amd64: + uses: ./.github/workflows/benchmark-template.yaml + with: + benchGitRef: release-1.3 diff --git a/.github/workflows/benchmark-template.yaml b/.github/workflows/benchmark-template.yaml index 057286b..2abadbf 100644 --- a/.github/workflows/benchmark-template.yaml +++ b/.github/workflows/benchmark-template.yaml @@ -1,57 +1,57 @@ ---- -name: Reusable Benchmark Template -on: - workflow_call: - inputs: - # which git reference to benchmark against - benchGitRef: - required: true - type: string - maxAcceptableDifferencePercent: - required: false - type: number - default: 5 - runs-on: - required: false - type: string - default: "['ubuntu-latest']" -permissions: read-all - -jobs: - benchmark: - runs-on: ${{ fromJson(inputs.runs-on) }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - name: Run Benchmarks - run: | - BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ inputs.benchGitRef }} - - run: | - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - cat result.txt >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" - cat <> "$GITHUB_STEP_SUMMARY" -
- The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10). - The last row has the Geometric Mean (geomean) for the given rows in the table. - Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help. - EOL - - name: Validate results under acceptable limit - run: | - export MAX_ACCEPTABLE_DIFFERENCE=${{ inputs.maxAcceptableDifferencePercent }} - while IFS= read -r line; do - # Get fourth value, which is the comparison with the base. - value="$(echo "$line" | awk '{print $4}')" - if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then - if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then - echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)" - exit 1 - fi - fi - done < <(grep geomean result.txt) +--- +name: Reusable Benchmark Template +on: + workflow_call: + inputs: + # which git reference to benchmark against + benchGitRef: + required: true + type: string + maxAcceptableDifferencePercent: + required: false + type: number + default: 5 + runs-on: + required: false + type: string + default: "['ubuntu-latest']" +permissions: read-all + +jobs: + benchmark: + runs-on: ${{ fromJson(inputs.runs-on) }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: Run Benchmarks + run: | + BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ inputs.benchGitRef }} + - run: | + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat result.txt >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat <> "$GITHUB_STEP_SUMMARY" +
+ The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10). + The last row has the Geometric Mean (geomean) for the given rows in the table. + Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help. + EOL + - name: Validate results under acceptable limit + run: | + export MAX_ACCEPTABLE_DIFFERENCE=${{ inputs.maxAcceptableDifferencePercent }} + while IFS= read -r line; do + # Get fourth value, which is the comparison with the base. + value="$(echo "$line" | awk '{print $4}')" + if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then + if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then + echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)" + exit 1 + fi + fi + done < <(grep geomean result.txt) diff --git a/.github/workflows/failpoint_test.yaml b/.github/workflows/failpoint_test.yaml index ce626ca..719da86 100644 --- a/.github/workflows/failpoint_test.yaml +++ b/.github/workflows/failpoint_test.yaml @@ -1,20 +1,20 @@ ---- -name: Failpoint test -on: [push, pull_request] -permissions: read-all -jobs: - test: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - run: | - make gofail-enable - make test-failpoint +--- +name: Failpoint test +on: [push, pull_request] +permissions: read-all +jobs: + test: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: | + make gofail-enable + make test-failpoint diff --git a/.github/workflows/gh-workflow-approve.yaml b/.github/workflows/gh-workflow-approve.yaml index 4a51970..0877f55 100644 --- a/.github/workflows/gh-workflow-approve.yaml +++ b/.github/workflows/gh-workflow-approve.yaml @@ -1,42 +1,42 @@ ---- -name: Approve GitHub Workflows -permissions: read-all -on: - pull_request_target: - types: - - labeled - - synchronize - branches: - - main - - release-1.3 - -jobs: - approve: - name: Approve ok-to-test - if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Update PR - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - continue-on-error: true - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} - script: | - const result = await github.rest.actions.listWorkflowRunsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - event: "pull_request", - status: "action_required", - head_sha: context.payload.pull_request.head.sha, - per_page: 100 - }); - for (var run of result.data.workflow_runs) { - await github.rest.actions.approveWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run.id - }); - } +--- +name: Approve GitHub Workflows +permissions: read-all +on: + pull_request_target: + types: + - labeled + - synchronize + branches: + - main + - release-1.3 + +jobs: + approve: + name: Approve ok-to-test + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Update PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} + script: | + const result = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + event: "pull_request", + status: "action_required", + head_sha: context.payload.pull_request.head.sha, + per_page: 100 + }); + for (var run of result.data.workflow_runs) { + await github.rest.actions.approveWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id + }); + } diff --git a/.github/workflows/robustness_nightly.yaml b/.github/workflows/robustness_nightly.yaml index df04e78..5aaf96f 100644 --- a/.github/workflows/robustness_nightly.yaml +++ b/.github/workflows/robustness_nightly.yaml @@ -1,17 +1,17 @@ ---- -name: Robustness Nightly -permissions: read-all -on: - schedule: - - cron: '25 9 * * *' # runs every day at 09:25 UTC - # workflow_dispatch enables manual testing of this job by maintainers - workflow_dispatch: - -jobs: - amd64: - # GHA has a maximum amount of 6h execution time, we try to get done within 3h - uses: ./.github/workflows/robustness_template.yaml - with: - count: 100 - testTimeout: 200m - runs-on: "['ubuntu-latest']" +--- +name: Robustness Nightly +permissions: read-all +on: + schedule: + - cron: '25 9 * * *' # runs every day at 09:25 UTC + # workflow_dispatch enables manual testing of this job by maintainers + workflow_dispatch: + +jobs: + amd64: + # GHA has a maximum amount of 6h execution time, we try to get done within 3h + uses: ./.github/workflows/robustness_template.yaml + with: + count: 100 + testTimeout: 200m + runs-on: "['ubuntu-latest']" diff --git a/.github/workflows/robustness_template.yaml b/.github/workflows/robustness_template.yaml index befe7df..0c146dd 100644 --- a/.github/workflows/robustness_template.yaml +++ b/.github/workflows/robustness_template.yaml @@ -1,50 +1,50 @@ ---- -name: Reusable Robustness Workflow -on: - workflow_call: - inputs: - count: - required: true - type: number - testTimeout: - required: false - type: string - default: '30m' - runs-on: - required: false - type: string - default: "['ubuntu-latest']" -permissions: read-all - -jobs: - test: - # this is to prevent the job to run at forked projects - if: github.repository == 'etcd-io/bbolt' - timeout-minutes: 210 - runs-on: ${{ fromJson(inputs.runs-on) }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - name: test-robustness - run: | - set -euo pipefail - sudo apt-get install -y dmsetup xfsprogs - - ROBUSTNESS_TESTFLAGS="--count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} -failfast" make test-robustness - - - name: Host Status - if: always() - run: | - set -x - mount - df - losetup -l - - name: Kernel Message - if: failure() - run: | - sudo lsmod - sudo dmesg -T -f kern +--- +name: Reusable Robustness Workflow +on: + workflow_call: + inputs: + count: + required: true + type: number + testTimeout: + required: false + type: string + default: '30m' + runs-on: + required: false + type: string + default: "['ubuntu-latest']" +permissions: read-all + +jobs: + test: + # this is to prevent the job to run at forked projects + if: github.repository == 'etcd-io/bbolt' + timeout-minutes: 210 + runs-on: ${{ fromJson(inputs.runs-on) }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: test-robustness + run: | + set -euo pipefail + sudo apt-get install -y dmsetup xfsprogs + + ROBUSTNESS_TESTFLAGS="--count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} -failfast" make test-robustness + + - name: Host Status + if: always() + run: | + set -x + mount + df + losetup -l + - name: Kernel Message + if: failure() + run: | + sudo lsmod + sudo dmesg -T -f kern diff --git a/.github/workflows/robustness_test.yaml b/.github/workflows/robustness_test.yaml index 635d4e8..975e892 100644 --- a/.github/workflows/robustness_test.yaml +++ b/.github/workflows/robustness_test.yaml @@ -1,16 +1,16 @@ -name: Robustness Test -on: [push, pull_request] -permissions: read-all -jobs: - amd64: - uses: ./.github/workflows/robustness_template.yaml - with: - count: 10 - testTimeout: 30m - runs-on: "['ubuntu-latest']" - arm64: - uses: ./.github/workflows/robustness_template.yaml - with: - count: 10 - testTimeout: 30m - runs-on: "['ubuntu-24.04-arm']" +name: Robustness Test +on: [push, pull_request] +permissions: read-all +jobs: + amd64: + uses: ./.github/workflows/robustness_template.yaml + with: + count: 10 + testTimeout: 30m + runs-on: "['ubuntu-latest']" + arm64: + uses: ./.github/workflows/robustness_template.yaml + with: + count: 10 + testTimeout: 30m + runs-on: "['ubuntu-24.04-arm']" diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 1abb63a..0eb7eec 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -1,19 +1,19 @@ -name: 'Close stale issues and PRs' -on: - schedule: - - cron: '0 0 * * *' # every day at 00:00 UTC - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 - with: - days-before-stale: 90 - days-before-close: 21 - stale-issue-label: stale - stale-pr-label: stale +name: 'Close stale issues and PRs' +on: + schedule: + - cron: '0 0 * * *' # every day at 00:00 UTC + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 + with: + days-before-stale: 90 + days-before-close: 21 + stale-issue-label: stale + stale-pr-label: stale diff --git a/.github/workflows/tests-template.yml b/.github/workflows/tests-template.yml index ad92c8c..c192f3f 100644 --- a/.github/workflows/tests-template.yml +++ b/.github/workflows/tests-template.yml @@ -1,55 +1,55 @@ ---- -name: Reusable unit test Workflow -on: - workflow_call: - inputs: - runs-on: - required: false - type: string - default: ubuntu-latest - targets: - required: false - type: string - default: "['linux-unit-test-1-cpu','linux-unit-test-2-cpu','linux-unit-test-4-cpu']" -permissions: read-all - -jobs: - test-linux: - strategy: - fail-fast: false - matrix: - target: ${{ fromJSON(inputs.targets) }} - runs-on: ${{ inputs.runs-on }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - run: make fmt - - env: - TARGET: ${{ matrix.target }} - run: | - case "${TARGET}" in - linux-unit-test-1-cpu) - CPU=1 make test - ;; - linux-unit-test-2-cpu) - CPU=2 make test - ;; - linux-unit-test-4-cpu) - CPU=4 make test - ;; - linux-unit-test-4-cpu-race) - CPU=4 ENABLE_RACE=true make test - ;; - *) - echo "Failed to find target" - exit 1 - ;; - esac - - name: golangci-lint - uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 - with: - version: v2.1.6 +--- +name: Reusable unit test Workflow +on: + workflow_call: + inputs: + runs-on: + required: false + type: string + default: ubuntu-latest + targets: + required: false + type: string + default: "['linux-unit-test-1-cpu','linux-unit-test-2-cpu','linux-unit-test-4-cpu']" +permissions: read-all + +jobs: + test-linux: + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(inputs.targets) }} + runs-on: ${{ inputs.runs-on }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: make fmt + - env: + TARGET: ${{ matrix.target }} + run: | + case "${TARGET}" in + linux-unit-test-1-cpu) + CPU=1 make test + ;; + linux-unit-test-2-cpu) + CPU=2 make test + ;; + linux-unit-test-4-cpu) + CPU=4 make test + ;; + linux-unit-test-4-cpu-race) + CPU=4 ENABLE_RACE=true make test + ;; + *) + echo "Failed to find target" + exit 1 + ;; + esac + - name: golangci-lint + uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 + with: + version: v2.1.6 diff --git a/.github/workflows/tests_amd64.yaml b/.github/workflows/tests_amd64.yaml index 7372dd7..a377890 100644 --- a/.github/workflows/tests_amd64.yaml +++ b/.github/workflows/tests_amd64.yaml @@ -1,26 +1,26 @@ ---- -name: Tests AMD64 -permissions: read-all -on: [push, pull_request] -jobs: - test-linux-amd64: - uses: ./.github/workflows/tests-template.yml - test-linux-amd64-race: - uses: ./.github/workflows/tests-template.yml - with: - runs-on: ubuntu-latest - targets: "['linux-unit-test-4-cpu-race']" - - coverage: - needs: - - test-linux-amd64 - - test-linux-amd64-race - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - run: make coverage +--- +name: Tests AMD64 +permissions: read-all +on: [push, pull_request] +jobs: + test-linux-amd64: + uses: ./.github/workflows/tests-template.yml + test-linux-amd64-race: + uses: ./.github/workflows/tests-template.yml + with: + runs-on: ubuntu-latest + targets: "['linux-unit-test-4-cpu-race']" + + coverage: + needs: + - test-linux-amd64 + - test-linux-amd64-race + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: make coverage diff --git a/.github/workflows/tests_arm64.yaml b/.github/workflows/tests_arm64.yaml index c89b322..e7cfc16 100644 --- a/.github/workflows/tests_arm64.yaml +++ b/.github/workflows/tests_arm64.yaml @@ -1,26 +1,26 @@ ---- -name: Tests ARM64 -permissions: read-all -on: [push, pull_request] -jobs: - test-linux-arm64: - uses: ./.github/workflows/tests-template.yml - test-linux-arm64-race: - uses: ./.github/workflows/tests-template.yml - with: - runs-on: ubuntu-24.04-arm - targets: "['linux-unit-test-4-cpu-race']" - - coverage: - needs: - - test-linux-arm64 - - test-linux-arm64-race - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - run: make coverage +--- +name: Tests ARM64 +permissions: read-all +on: [push, pull_request] +jobs: + test-linux-arm64: + uses: ./.github/workflows/tests-template.yml + test-linux-arm64-race: + uses: ./.github/workflows/tests-template.yml + with: + runs-on: ubuntu-24.04-arm + targets: "['linux-unit-test-4-cpu-race']" + + coverage: + needs: + - test-linux-arm64 + - test-linux-arm64-race + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: make coverage diff --git a/.github/workflows/tests_windows.yml b/.github/workflows/tests_windows.yml index 54546e1..133ff4d 100644 --- a/.github/workflows/tests_windows.yml +++ b/.github/workflows/tests_windows.yml @@ -1,57 +1,57 @@ ---- -name: Tests -on: [push, pull_request] -permissions: read-all -jobs: - test-windows: - strategy: - fail-fast: false - matrix: - target: - - windows-amd64-unit-test-4-cpu - # FIXME(fuweid): - # - # The windows will throws the following error when enable race. - # We skip it until we have solution. - # - # ThreadSanitizer failed to allocate 0x000200000000 (8589934592) bytes at 0x0400c0000000 (error code: 1455) - # - # - windows-amd64-unit-test-4-cpu-race - runs-on: windows-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - run: make fmt - - env: - TARGET: ${{ matrix.target }} - run: | - case "${TARGET}" in - windows-amd64-unit-test-4-cpu) - CPU=4 make test - ;; - *) - echo "Failed to find target" - exit 1 - ;; - esac - shell: bash - - name: golangci-lint - uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 - with: - version: v2.1.6 - - coverage: - needs: ["test-windows"] - runs-on: windows-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: goversion - run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: ${{ steps.goversion.outputs.goversion }} - - run: make coverage +--- +name: Tests +on: [push, pull_request] +permissions: read-all +jobs: + test-windows: + strategy: + fail-fast: false + matrix: + target: + - windows-amd64-unit-test-4-cpu + # FIXME(fuweid): + # + # The windows will throws the following error when enable race. + # We skip it until we have solution. + # + # ThreadSanitizer failed to allocate 0x000200000000 (8589934592) bytes at 0x0400c0000000 (error code: 1455) + # + # - windows-amd64-unit-test-4-cpu-race + runs-on: windows-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: make fmt + - env: + TARGET: ${{ matrix.target }} + run: | + case "${TARGET}" in + windows-amd64-unit-test-4-cpu) + CPU=4 make test + ;; + *) + echo "Failed to find target" + exit 1 + ;; + esac + shell: bash + - name: golangci-lint + uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 + with: + version: v2.1.6 + + coverage: + needs: ["test-windows"] + runs-on: windows-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: make coverage diff --git a/.gitignore b/.gitignore index ed4d259..6cab0c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,12 @@ -*.prof -*.test -*.swp -/bin/ -cover.out -cover-*.out -/.idea -*.iml -/bbolt -/cmd/bbolt/bbolt -.DS_Store - +*.prof +*.test +*.swp +/bin/ +cover.out +cover-*.out +/.idea +*.iml +/bbolt +/cmd/bbolt/bbolt +.DS_Store + diff --git a/.go-version b/.go-version index ae96cc7..1b5f24e 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.24.3 +1.24.3 diff --git a/.golangci.yaml b/.golangci.yaml index 68fc131..6ad3008 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,34 +1,34 @@ -formatters: - enable: - - gofmt - - goimports - settings: # please keep this alphabetized - goimports: - local-prefixes: - - go.etcd.io # Put imports beginning with prefix after 3rd-party packages. -issues: - max-same-issues: 0 -linters: - default: none - enable: # please keep this alphabetized - - errcheck - - govet - - ineffassign - - staticcheck - - unused - exclusions: - presets: - - comments - - common-false-positives - - legacy - - std-error-handling - settings: # please keep this alphabetized - staticcheck: - checks: - - all - - -QF1003 # Convert if/else-if chain to tagged switch - - -QF1010 # Convert slice of bytes to string when printing it - - -ST1003 # Poorly chosen identifier - - -ST1005 # Incorrectly formatted error string - - -ST1012 # Poorly chosen name for error variable -version: "2" +formatters: + enable: + - gofmt + - goimports + settings: # please keep this alphabetized + goimports: + local-prefixes: + - go.etcd.io # Put imports beginning with prefix after 3rd-party packages. +issues: + max-same-issues: 0 +linters: + default: none + enable: # please keep this alphabetized + - errcheck + - govet + - ineffassign + - staticcheck + - unused + exclusions: + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + settings: # please keep this alphabetized + staticcheck: + checks: + - all + - -QF1003 # Convert if/else-if chain to tagged switch + - -QF1010 # Convert slice of bytes to string when printing it + - -ST1003 # Poorly chosen identifier + - -ST1005 # Incorrectly formatted error string + - -ST1012 # Poorly chosen name for error variable +version: "2" diff --git a/CHANGELOG/CHANGELOG-1.3.md b/CHANGELOG/CHANGELOG-1.3.md index 23009eb..3afae77 100644 --- a/CHANGELOG/CHANGELOG-1.3.md +++ b/CHANGELOG/CHANGELOG-1.3.md @@ -1,90 +1,90 @@ -Note that we start to track changes starting from v1.3.7. - -
- -## v1.3.11(2024-08-21) - -### BoltDB -- Fix [the `freelist.allocs` isn't rollbacked when a tx is rollbacked](https://github.com/etcd-io/bbolt/pull/823). - -### CMD -- Add [`-gobench-output` option for bench command to adapt to benchstat](https://github.com/etcd-io/bbolt/pull/802). - -### Other -- [Bump go version to 1.22.x](https://github.com/etcd-io/bbolt/pull/822). -- This patch also added `dmflakey` package, which can be reused by other projects. See https://github.com/etcd-io/bbolt/pull/812. - -
- -## v1.3.10(2024-05-06) - -### BoltDB -- [Remove deprecated `UnsafeSlice` and use `unsafe.Slice`](https://github.com/etcd-io/bbolt/pull/717) -- [Stabilize the behaviour of Prev when the cursor already points to the first element](https://github.com/etcd-io/bbolt/pull/744) - -### Other -- [Bump go version to 1.21.9](https://github.com/etcd-io/bbolt/pull/713) - -
- -## v1.3.9(2024-02-24) - -### BoltDB -- [Clone the key before operating data in bucket against the key](https://github.com/etcd-io/bbolt/pull/639) - -### CMD -- [Fix `bbolt keys` and `bbolt get` to prevent them from panicking when no parameter provided](https://github.com/etcd-io/bbolt/pull/683) - -
- -## v1.3.8(2023-10-26) - -### BoltDB -- Fix [db.close() doesn't unlock the db file if db.munnmap() fails](https://github.com/etcd-io/bbolt/pull/439). -- [Avoid syscall.Syscall use on OpenBSD](https://github.com/etcd-io/bbolt/pull/406). -- Fix [rollback panicking after mlock failed or both meta pages corrupted](https://github.com/etcd-io/bbolt/pull/444). -- Fix [bbolt panicking due to 64bit unaligned on arm32](https://github.com/etcd-io/bbolt/pull/584). - -### CMD -- [Update the usage of surgery command](https://github.com/etcd-io/bbolt/pull/411). - -
- -## v1.3.7(2023-01-31) - -### BoltDB -- Add [recursive checker to confirm database consistency](https://github.com/etcd-io/bbolt/pull/225). -- Add [support to get the page size from the second meta page if the first one is invalid](https://github.com/etcd-io/bbolt/pull/294). -- Add [support for loong64 arch](https://github.com/etcd-io/bbolt/pull/303). -- Add [internal iterator to Bucket that goes over buckets](https://github.com/etcd-io/bbolt/pull/356). -- Add [validation on page read and write](https://github.com/etcd-io/bbolt/pull/358). -- Add [PreLoadFreelist option to support loading free pages in readonly mode](https://github.com/etcd-io/bbolt/pull/381). -- Add [(*Tx) CheckWithOption to support generating human-readable diagnostic messages](https://github.com/etcd-io/bbolt/pull/395). -- Fix [Use `golang.org/x/sys/windows` for `FileLockEx`/`UnlockFileEx`](https://github.com/etcd-io/bbolt/pull/283). -- Fix [readonly file mapping on windows](https://github.com/etcd-io/bbolt/pull/307). -- Fix [the "Last" method might return no data due to not skipping the empty pages](https://github.com/etcd-io/bbolt/pull/341). -- Fix [panic on db.meta when rollback](https://github.com/etcd-io/bbolt/pull/362). - -### CMD -- Add [support for get keys in sub buckets in `bbolt get` command](https://github.com/etcd-io/bbolt/pull/295). -- Add [support for `--format` flag for `bbolt keys` command](https://github.com/etcd-io/bbolt/pull/306). -- Add [safeguards to bbolt CLI commands](https://github.com/etcd-io/bbolt/pull/354). -- Add [`bbolt page` supports --all and --value-format=redacted formats](https://github.com/etcd-io/bbolt/pull/359). -- Add [`bbolt surgery` commands](https://github.com/etcd-io/bbolt/issues/370). -- Fix [open db file readonly mode for commands which shouldn't update the db file](https://github.com/etcd-io/bbolt/pull/365), see also [pull/292](https://github.com/etcd-io/bbolt/pull/292). - -### Other -- [Build bbolt CLI tool, test and format the source code using golang 1.17.13](https://github.com/etcd-io/bbolt/pull/297). -- [Bump golang.org/x/sys to v0.4.0](https://github.com/etcd-io/bbolt/pull/397). - -### Summary -Release v1.3.7 contains following critical fixes: -- fix to problem that `Last` method might return incorrect value ([#341](https://github.com/etcd-io/bbolt/pull/341)) -- fix of potential panic when performing transaction's rollback ([#362](https://github.com/etcd-io/bbolt/pull/362)) - -Other changes focused on defense-in-depth ([#358](https://github.com/etcd-io/bbolt/pull/358), [#294](https://github.com/etcd-io/bbolt/pull/294), [#225](https://github.com/etcd-io/bbolt/pull/225), [#395](https://github.com/etcd-io/bbolt/pull/395)) - -`bbolt` command line tool was expanded to: -- allow fixing simple corruptions by `bbolt surgery` ([#370](https://github.com/etcd-io/bbolt/pull/370)) -- be flexible about output formatting ([#306](https://github.com/etcd-io/bbolt/pull/306), [#359](https://github.com/etcd-io/bbolt/pull/359)) -- allow accessing data in subbuckets ([#295](https://github.com/etcd-io/bbolt/pull/295)) +Note that we start to track changes starting from v1.3.7. + +
+ +## v1.3.11(2024-08-21) + +### BoltDB +- Fix [the `freelist.allocs` isn't rollbacked when a tx is rollbacked](https://github.com/etcd-io/bbolt/pull/823). + +### CMD +- Add [`-gobench-output` option for bench command to adapt to benchstat](https://github.com/etcd-io/bbolt/pull/802). + +### Other +- [Bump go version to 1.22.x](https://github.com/etcd-io/bbolt/pull/822). +- This patch also added `dmflakey` package, which can be reused by other projects. See https://github.com/etcd-io/bbolt/pull/812. + +
+ +## v1.3.10(2024-05-06) + +### BoltDB +- [Remove deprecated `UnsafeSlice` and use `unsafe.Slice`](https://github.com/etcd-io/bbolt/pull/717) +- [Stabilize the behaviour of Prev when the cursor already points to the first element](https://github.com/etcd-io/bbolt/pull/744) + +### Other +- [Bump go version to 1.21.9](https://github.com/etcd-io/bbolt/pull/713) + +
+ +## v1.3.9(2024-02-24) + +### BoltDB +- [Clone the key before operating data in bucket against the key](https://github.com/etcd-io/bbolt/pull/639) + +### CMD +- [Fix `bbolt keys` and `bbolt get` to prevent them from panicking when no parameter provided](https://github.com/etcd-io/bbolt/pull/683) + +
+ +## v1.3.8(2023-10-26) + +### BoltDB +- Fix [db.close() doesn't unlock the db file if db.munnmap() fails](https://github.com/etcd-io/bbolt/pull/439). +- [Avoid syscall.Syscall use on OpenBSD](https://github.com/etcd-io/bbolt/pull/406). +- Fix [rollback panicking after mlock failed or both meta pages corrupted](https://github.com/etcd-io/bbolt/pull/444). +- Fix [bbolt panicking due to 64bit unaligned on arm32](https://github.com/etcd-io/bbolt/pull/584). + +### CMD +- [Update the usage of surgery command](https://github.com/etcd-io/bbolt/pull/411). + +
+ +## v1.3.7(2023-01-31) + +### BoltDB +- Add [recursive checker to confirm database consistency](https://github.com/etcd-io/bbolt/pull/225). +- Add [support to get the page size from the second meta page if the first one is invalid](https://github.com/etcd-io/bbolt/pull/294). +- Add [support for loong64 arch](https://github.com/etcd-io/bbolt/pull/303). +- Add [internal iterator to Bucket that goes over buckets](https://github.com/etcd-io/bbolt/pull/356). +- Add [validation on page read and write](https://github.com/etcd-io/bbolt/pull/358). +- Add [PreLoadFreelist option to support loading free pages in readonly mode](https://github.com/etcd-io/bbolt/pull/381). +- Add [(*Tx) CheckWithOption to support generating human-readable diagnostic messages](https://github.com/etcd-io/bbolt/pull/395). +- Fix [Use `golang.org/x/sys/windows` for `FileLockEx`/`UnlockFileEx`](https://github.com/etcd-io/bbolt/pull/283). +- Fix [readonly file mapping on windows](https://github.com/etcd-io/bbolt/pull/307). +- Fix [the "Last" method might return no data due to not skipping the empty pages](https://github.com/etcd-io/bbolt/pull/341). +- Fix [panic on db.meta when rollback](https://github.com/etcd-io/bbolt/pull/362). + +### CMD +- Add [support for get keys in sub buckets in `bbolt get` command](https://github.com/etcd-io/bbolt/pull/295). +- Add [support for `--format` flag for `bbolt keys` command](https://github.com/etcd-io/bbolt/pull/306). +- Add [safeguards to bbolt CLI commands](https://github.com/etcd-io/bbolt/pull/354). +- Add [`bbolt page` supports --all and --value-format=redacted formats](https://github.com/etcd-io/bbolt/pull/359). +- Add [`bbolt surgery` commands](https://github.com/etcd-io/bbolt/issues/370). +- Fix [open db file readonly mode for commands which shouldn't update the db file](https://github.com/etcd-io/bbolt/pull/365), see also [pull/292](https://github.com/etcd-io/bbolt/pull/292). + +### Other +- [Build bbolt CLI tool, test and format the source code using golang 1.17.13](https://github.com/etcd-io/bbolt/pull/297). +- [Bump golang.org/x/sys to v0.4.0](https://github.com/etcd-io/bbolt/pull/397). + +### Summary +Release v1.3.7 contains following critical fixes: +- fix to problem that `Last` method might return incorrect value ([#341](https://github.com/etcd-io/bbolt/pull/341)) +- fix of potential panic when performing transaction's rollback ([#362](https://github.com/etcd-io/bbolt/pull/362)) + +Other changes focused on defense-in-depth ([#358](https://github.com/etcd-io/bbolt/pull/358), [#294](https://github.com/etcd-io/bbolt/pull/294), [#225](https://github.com/etcd-io/bbolt/pull/225), [#395](https://github.com/etcd-io/bbolt/pull/395)) + +`bbolt` command line tool was expanded to: +- allow fixing simple corruptions by `bbolt surgery` ([#370](https://github.com/etcd-io/bbolt/pull/370)) +- be flexible about output formatting ([#306](https://github.com/etcd-io/bbolt/pull/306), [#359](https://github.com/etcd-io/bbolt/pull/359)) +- allow accessing data in subbuckets ([#295](https://github.com/etcd-io/bbolt/pull/295)) diff --git a/CHANGELOG/CHANGELOG-1.4.md b/CHANGELOG/CHANGELOG-1.4.md index f5cb940..9bec16e 100644 --- a/CHANGELOG/CHANGELOG-1.4.md +++ b/CHANGELOG/CHANGELOG-1.4.md @@ -1,76 +1,76 @@ - -
- -## v1.4.0(2025-02-05) -There isn't any production code change since v1.4.0-beta.0. Only some dependencies -are bumped, also updated some typos in comment and readme, and removed the legacy -build tag `// +build` in https://github.com/etcd-io/bbolt/pull/879. - -
- -## v1.4.0-beta.0(2024-11-04) - -### BoltDB -- Reorganized the directory structure of freelist source code - - [Move array related freelist source code into a separate file](https://github.com/etcd-io/bbolt/pull/777) - - [Move method `freePages` into freelist.go](https://github.com/etcd-io/bbolt/pull/783) - - [Add an interface for freelist](https://github.com/etcd-io/bbolt/pull/775) -- [Rollback alloc map when a transaction is rollbacked](https://github.com/etcd-io/bbolt/pull/819) -- [No handling freelist as a special case when freeing a page](https://github.com/etcd-io/bbolt/pull/788) -- [Ensure hashmap init method clears the data structures](https://github.com/etcd-io/bbolt/pull/794) -- [Panicking when a write transaction tries to free a page allocated by itself](https://github.com/etcd-io/bbolt/pull/792) - -### CMD -- [Add `-gobench-output` flag for `bbolt bench` command](https://github.com/etcd-io/bbolt/pull/765) - -### Other -- [Bump go version to 1.23.x](https://github.com/etcd-io/bbolt/pull/821) - -
- -## v1.4.0-alpha.1(2024-05-06) - -### BoltDB -- [Enhance check functionality to support checking starting from a pageId](https://github.com/etcd-io/bbolt/pull/659) -- [Optimize the logger performance for frequent called methods](https://github.com/etcd-io/bbolt/pull/741) -- [Stabilize the behaviour of Prev when the cursor already points to the first element](https://github.com/etcd-io/bbolt/pull/734) - -### CMD -- [Fix `bbolt keys` and `bbolt get` to prevent them from panicking when no parameter provided](https://github.com/etcd-io/bbolt/pull/682) -- [Fix surgery freelist command in info logs](https://github.com/etcd-io/bbolt/pull/700) -- [Remove txid references in surgery meta command's comment and description](https://github.com/etcd-io/bbolt/pull/703) -- [Add rnd read capabilities to bbolt bench](https://github.com/etcd-io/bbolt/pull/711) -- [Use `cobra.ExactArgs` to simplify the argument number check](https://github.com/etcd-io/bbolt/pull/728) -- [Migrate `bbolt check` command to cobra style](https://github.com/etcd-io/bbolt/pull/723) -- [Simplify the naming of cobra commands](https://github.com/etcd-io/bbolt/pull/732) -- [Aggregate adding completed ops for read test of the `bbolt bench` command](https://github.com/etcd-io/bbolt/pull/721) -- [Add `--from-page` flag to `bbolt check` command](https://github.com/etcd-io/bbolt/pull/737) - -### Document -- [Add document for a known issue on the writing a value with a length of 0](https://github.com/etcd-io/bbolt/pull/730) - -### Test -- [Enhance robustness test to cover XFS](https://github.com/etcd-io/bbolt/pull/707) - -### Other -- [Bump go toolchain version to 1.22.2](https://github.com/etcd-io/bbolt/pull/712) - -
- -## v1.4.0-alpha.0(2024-01-12) - -### BoltDB -- [Improve the performance of hashmapGetFreePageIDs](https://github.com/etcd-io/bbolt/pull/419) -- [Improve CreateBucketIfNotExists to avoid double searching the same key](https://github.com/etcd-io/bbolt/pull/532) -- [Support Android platform](https://github.com/etcd-io/bbolt/pull/571) -- [Record the count of free page to improve the performance of hashmapFreeCount](https://github.com/etcd-io/bbolt/pull/585) -- [Add logger to bbolt](https://github.com/etcd-io/bbolt/issues/509) -- [Support moving bucket inside the same db](https://github.com/etcd-io/bbolt/pull/635) -- [Support inspecting database structure](https://github.com/etcd-io/bbolt/pull/674) - -### CMD -- [Add `surgery clear-page-elements` command](https://github.com/etcd-io/bbolt/pull/417) -- [Add `surgery abandon-freelist` command](https://github.com/etcd-io/bbolt/pull/443) -- [Add `bbolt version` command](https://github.com/etcd-io/bbolt/pull/552) -- [Add `bbolt inspect` command](https://github.com/etcd-io/bbolt/pull/674) -- [Add `--no-sync` option to `bbolt compact` command](https://github.com/etcd-io/bbolt/pull/290) + +
+ +## v1.4.0(2025-02-05) +There isn't any production code change since v1.4.0-beta.0. Only some dependencies +are bumped, also updated some typos in comment and readme, and removed the legacy +build tag `// +build` in https://github.com/etcd-io/bbolt/pull/879. + +
+ +## v1.4.0-beta.0(2024-11-04) + +### BoltDB +- Reorganized the directory structure of freelist source code + - [Move array related freelist source code into a separate file](https://github.com/etcd-io/bbolt/pull/777) + - [Move method `freePages` into freelist.go](https://github.com/etcd-io/bbolt/pull/783) + - [Add an interface for freelist](https://github.com/etcd-io/bbolt/pull/775) +- [Rollback alloc map when a transaction is rollbacked](https://github.com/etcd-io/bbolt/pull/819) +- [No handling freelist as a special case when freeing a page](https://github.com/etcd-io/bbolt/pull/788) +- [Ensure hashmap init method clears the data structures](https://github.com/etcd-io/bbolt/pull/794) +- [Panicking when a write transaction tries to free a page allocated by itself](https://github.com/etcd-io/bbolt/pull/792) + +### CMD +- [Add `-gobench-output` flag for `bbolt bench` command](https://github.com/etcd-io/bbolt/pull/765) + +### Other +- [Bump go version to 1.23.x](https://github.com/etcd-io/bbolt/pull/821) + +
+ +## v1.4.0-alpha.1(2024-05-06) + +### BoltDB +- [Enhance check functionality to support checking starting from a pageId](https://github.com/etcd-io/bbolt/pull/659) +- [Optimize the logger performance for frequent called methods](https://github.com/etcd-io/bbolt/pull/741) +- [Stabilize the behaviour of Prev when the cursor already points to the first element](https://github.com/etcd-io/bbolt/pull/734) + +### CMD +- [Fix `bbolt keys` and `bbolt get` to prevent them from panicking when no parameter provided](https://github.com/etcd-io/bbolt/pull/682) +- [Fix surgery freelist command in info logs](https://github.com/etcd-io/bbolt/pull/700) +- [Remove txid references in surgery meta command's comment and description](https://github.com/etcd-io/bbolt/pull/703) +- [Add rnd read capabilities to bbolt bench](https://github.com/etcd-io/bbolt/pull/711) +- [Use `cobra.ExactArgs` to simplify the argument number check](https://github.com/etcd-io/bbolt/pull/728) +- [Migrate `bbolt check` command to cobra style](https://github.com/etcd-io/bbolt/pull/723) +- [Simplify the naming of cobra commands](https://github.com/etcd-io/bbolt/pull/732) +- [Aggregate adding completed ops for read test of the `bbolt bench` command](https://github.com/etcd-io/bbolt/pull/721) +- [Add `--from-page` flag to `bbolt check` command](https://github.com/etcd-io/bbolt/pull/737) + +### Document +- [Add document for a known issue on the writing a value with a length of 0](https://github.com/etcd-io/bbolt/pull/730) + +### Test +- [Enhance robustness test to cover XFS](https://github.com/etcd-io/bbolt/pull/707) + +### Other +- [Bump go toolchain version to 1.22.2](https://github.com/etcd-io/bbolt/pull/712) + +
+ +## v1.4.0-alpha.0(2024-01-12) + +### BoltDB +- [Improve the performance of hashmapGetFreePageIDs](https://github.com/etcd-io/bbolt/pull/419) +- [Improve CreateBucketIfNotExists to avoid double searching the same key](https://github.com/etcd-io/bbolt/pull/532) +- [Support Android platform](https://github.com/etcd-io/bbolt/pull/571) +- [Record the count of free page to improve the performance of hashmapFreeCount](https://github.com/etcd-io/bbolt/pull/585) +- [Add logger to bbolt](https://github.com/etcd-io/bbolt/issues/509) +- [Support moving bucket inside the same db](https://github.com/etcd-io/bbolt/pull/635) +- [Support inspecting database structure](https://github.com/etcd-io/bbolt/pull/674) + +### CMD +- [Add `surgery clear-page-elements` command](https://github.com/etcd-io/bbolt/pull/417) +- [Add `surgery abandon-freelist` command](https://github.com/etcd-io/bbolt/pull/443) +- [Add `bbolt version` command](https://github.com/etcd-io/bbolt/pull/552) +- [Add `bbolt inspect` command](https://github.com/etcd-io/bbolt/pull/674) +- [Add `--no-sync` option to `bbolt compact` command](https://github.com/etcd-io/bbolt/pull/290) diff --git a/LICENSE b/LICENSE index 004e77f..42acf94 100644 --- a/LICENSE +++ b/LICENSE @@ -1,20 +1,20 @@ -The MIT License (MIT) - -Copyright (c) 2013 Ben Johnson - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +The MIT License (MIT) + +Copyright (c) 2013 Ben Johnson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile index f5a6703..a79d245 100644 --- a/Makefile +++ b/Makefile @@ -1,108 +1,108 @@ -BRANCH=`git rev-parse --abbrev-ref HEAD` -COMMIT=`git rev-parse --short HEAD` -GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" -GOFILES = $(shell find . -name \*.go) - -TESTFLAGS_RACE=-race=false -ifdef ENABLE_RACE - TESTFLAGS_RACE=-race=true -endif - -TESTFLAGS_CPU= -ifdef CPU - TESTFLAGS_CPU=-cpu=$(CPU) -endif -TESTFLAGS = $(TESTFLAGS_RACE) $(TESTFLAGS_CPU) $(EXTRA_TESTFLAGS) - -TESTFLAGS_TIMEOUT=30m -ifdef TIMEOUT - TESTFLAGS_TIMEOUT=$(TIMEOUT) -endif - -TESTFLAGS_ENABLE_STRICT_MODE=false -ifdef ENABLE_STRICT_MODE - TESTFLAGS_ENABLE_STRICT_MODE=$(ENABLE_STRICT_MODE) -endif - -.EXPORT_ALL_VARIABLES: -TEST_ENABLE_STRICT_MODE=${TESTFLAGS_ENABLE_STRICT_MODE} - -.PHONY: fmt -fmt: - @echo "Verifying gofmt, failures can be fixed with ./scripts/fix.sh" - @!(gofmt -l -s -d ${GOFILES} | grep '[a-z]') - - @echo "Verifying goimports, failures can be fixed with ./scripts/fix.sh" - @!(go run golang.org/x/tools/cmd/goimports@latest -l -d ${GOFILES} | grep '[a-z]') - -.PHONY: lint -lint: - golangci-lint run ./... - -.PHONY: test -test: - @echo "hashmap freelist test" - BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} - BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./internal/... - BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt - - @echo "array freelist test" - BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} - BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/... - BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt - -.PHONY: coverage -coverage: - @echo "hashmap freelist test" - TEST_FREELIST_TYPE=hashmap go test -v -timeout ${TESTFLAGS_TIMEOUT} \ - -coverprofile cover-freelist-hashmap.out -covermode atomic - - @echo "array freelist test" - TEST_FREELIST_TYPE=array go test -v -timeout ${TESTFLAGS_TIMEOUT} \ - -coverprofile cover-freelist-array.out -covermode atomic - -BOLT_CMD=bbolt - -build: - go build -o bin/${BOLT_CMD} ./cmd/${BOLT_CMD} - -.PHONY: clean -clean: # Clean binaries - rm -f ./bin/${BOLT_CMD} - -.PHONY: gofail-enable -gofail-enable: install-gofail - gofail enable . - -.PHONY: gofail-disable -gofail-disable: install-gofail - gofail disable . - -.PHONY: install-gofail -install-gofail: - go install go.etcd.io/gofail - -.PHONY: test-failpoint -test-failpoint: - @echo "[failpoint] hashmap freelist test" - BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint - - @echo "[failpoint] array freelist test" - BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint - -.PHONY: test-robustness # Running robustness tests requires root permission for now -# TODO: Remove sudo once we fully migrate to the prow infrastructure -test-robustness: gofail-enable build - sudo env PATH=$$PATH go test -v ${TESTFLAGS} ./tests/dmflakey -test.root - sudo env PATH=$(PWD)/bin:$$PATH go test -v ${TESTFLAGS} ${ROBUSTNESS_TESTFLAGS} ./tests/robustness -test.root - -.PHONY: test-benchmark-compare -# Runs benchmark tests on the current git ref and the given REF, and compares -# the two. -test-benchmark-compare: install-benchstat - @git fetch - ./scripts/compare_benchmarks.sh $(REF) - -.PHONY: install-benchstat -install-benchstat: - go install golang.org/x/perf/cmd/benchstat@latest +BRANCH=`git rev-parse --abbrev-ref HEAD` +COMMIT=`git rev-parse --short HEAD` +GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" +GOFILES = $(shell find . -name \*.go) + +TESTFLAGS_RACE=-race=false +ifdef ENABLE_RACE + TESTFLAGS_RACE=-race=true +endif + +TESTFLAGS_CPU= +ifdef CPU + TESTFLAGS_CPU=-cpu=$(CPU) +endif +TESTFLAGS = $(TESTFLAGS_RACE) $(TESTFLAGS_CPU) $(EXTRA_TESTFLAGS) + +TESTFLAGS_TIMEOUT=30m +ifdef TIMEOUT + TESTFLAGS_TIMEOUT=$(TIMEOUT) +endif + +TESTFLAGS_ENABLE_STRICT_MODE=false +ifdef ENABLE_STRICT_MODE + TESTFLAGS_ENABLE_STRICT_MODE=$(ENABLE_STRICT_MODE) +endif + +.EXPORT_ALL_VARIABLES: +TEST_ENABLE_STRICT_MODE=${TESTFLAGS_ENABLE_STRICT_MODE} + +.PHONY: fmt +fmt: + @echo "Verifying gofmt, failures can be fixed with ./scripts/fix.sh" + @!(gofmt -l -s -d ${GOFILES} | grep '[a-z]') + + @echo "Verifying goimports, failures can be fixed with ./scripts/fix.sh" + @!(go run golang.org/x/tools/cmd/goimports@latest -l -d ${GOFILES} | grep '[a-z]') + +.PHONY: lint +lint: + golangci-lint run ./... + +.PHONY: test +test: + @echo "hashmap freelist test" + BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} + BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./internal/... + BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt + + @echo "array freelist test" + BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} + BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/... + BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt + +.PHONY: coverage +coverage: + @echo "hashmap freelist test" + TEST_FREELIST_TYPE=hashmap go test -v -timeout ${TESTFLAGS_TIMEOUT} \ + -coverprofile cover-freelist-hashmap.out -covermode atomic + + @echo "array freelist test" + TEST_FREELIST_TYPE=array go test -v -timeout ${TESTFLAGS_TIMEOUT} \ + -coverprofile cover-freelist-array.out -covermode atomic + +BOLT_CMD=bbolt + +build: + go build -o bin/${BOLT_CMD} ./cmd/${BOLT_CMD} + +.PHONY: clean +clean: # Clean binaries + rm -f ./bin/${BOLT_CMD} + +.PHONY: gofail-enable +gofail-enable: install-gofail + gofail enable . + +.PHONY: gofail-disable +gofail-disable: install-gofail + gofail disable . + +.PHONY: install-gofail +install-gofail: + go install go.etcd.io/gofail + +.PHONY: test-failpoint +test-failpoint: + @echo "[failpoint] hashmap freelist test" + BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint + + @echo "[failpoint] array freelist test" + BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint + +.PHONY: test-robustness # Running robustness tests requires root permission for now +# TODO: Remove sudo once we fully migrate to the prow infrastructure +test-robustness: gofail-enable build + sudo env PATH=$$PATH go test -v ${TESTFLAGS} ./tests/dmflakey -test.root + sudo env PATH=$(PWD)/bin:$$PATH go test -v ${TESTFLAGS} ${ROBUSTNESS_TESTFLAGS} ./tests/robustness -test.root + +.PHONY: test-benchmark-compare +# Runs benchmark tests on the current git ref and the given REF, and compares +# the two. +test-benchmark-compare: install-benchstat + @git fetch + ./scripts/compare_benchmarks.sh $(REF) + +.PHONY: install-benchstat +install-benchstat: + go install golang.org/x/perf/cmd/benchstat@latest diff --git a/OWNERS b/OWNERS index 91f168a..1bcc257 100644 --- a/OWNERS +++ b/OWNERS @@ -1,10 +1,10 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: - - ahrtr # Benjamin Wang - - serathius # Marek Siarkowicz - - ptabor # Piotr Tabor - - spzala # Sahdev Zala -reviewers: - - fuweid # Wei Fu - - tjungblu # Thomas Jungblut +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - ahrtr # Benjamin Wang + - serathius # Marek Siarkowicz + - ptabor # Piotr Tabor + - spzala # Sahdev Zala +reviewers: + - fuweid # Wei Fu + - tjungblu # Thomas Jungblut diff --git a/cmd/bbolt/OWNERS b/cmd/bbolt/OWNERS index d4d42d4..a0fe605 100644 --- a/cmd/bbolt/OWNERS +++ b/cmd/bbolt/OWNERS @@ -1,12 +1,12 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: - - ahrtr # Benjamin Wang - - fuweid # Wei Fu - - serathius # Marek Siarkowicz - - ptabor # Piotr Tabor - - spzala # Sahdev Zala - - tjungblu # Thomas Jungblut -reviewers: - - elbehery # Mustafa Elbehery - - ivanvc # Ivan Valdes +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - ahrtr # Benjamin Wang + - fuweid # Wei Fu + - serathius # Marek Siarkowicz + - ptabor # Piotr Tabor + - spzala # Sahdev Zala + - tjungblu # Thomas Jungblut +reviewers: + - elbehery # Mustafa Elbehery + - ivanvc # Ivan Valdes diff --git a/cmd/bbolt/README.md b/cmd/bbolt/README.md index a075849..7d9d47d 100644 --- a/cmd/bbolt/README.md +++ b/cmd/bbolt/README.md @@ -1,453 +1,453 @@ -# Introduction to bbolt command line - -`bbolt` provides a command line utility for inspecting and manipulating bbolt database files. To install bbolt command-line please refer [here](https://github.com/etcd-io/bbolt#installing) - -**Note**: [etcd](https://github.com/etcd-io/etcd) uses bbolt as its backend storage engine. In this document, we take etcd as an example to demonstrate the usage of bbolt commands. Refer to [install etcd](https://etcd.io/docs/v3.5/install/) for installing etcd. - -1. Start a single member etcd cluster with this command below: - - ```bash - $etcd - ``` - - It will create a directory `default.etcd` by default under current working directory, and the directory structure will look like this: - - ```bash - $tree default.etcd - default.etcd - └── member - ├── snap - │   └── db // this is bbolt database file - └── wal - └── 0000000000000000-0000000000000000.wal - - 3 directories, 2 files - ``` - -2. Put some dummy data using [etcdctl](https://github.com/etcd-io/etcd/tree/main/etcdctl). -3. Stop the etcd instance. Note a bbolt database file can only be opened by one read-write process, because it is exclusively locked when opened. - -## Usage - -- `bbolt command [arguments]` - -### help - -- help will print information about that command - - ```bash - $bbolt help - - The commands are: - - version prints the current version of bbolt - bench run synthetic benchmark against bbolt - buckets print a list of buckets - check verifies integrity of bbolt database - compact copies a bbolt database, compacting it in the process - dump print a hexadecimal dump of a single page - get print the value of a key in a bucket - info print basic info - keys print a list of keys in a bucket - help print this screen - page print one or more pages in human readable format - pages print list of pages with their types - page-item print the key and value of a page item. - stats iterate over all pages and generate usage stats - surgery perform surgery on bbolt database - ``` - -- you can use `help` with any command: `bbolt [command] -h` for more information about command. - -## Analyse bbolt database with bbolt command line - -### version - -- `version` print the current version information of bbolt command-line. -- usage: - `bbolt version` - - Example: - - ```bash - $bbolt version - bbolt version: 1.3.7 - Go Version: go1.21.6 - Go OS/Arch: darwin/arm64 - ``` - -### info - -- `info` print the basic information about the given Bbolt database. -- usage: - `bbolt info [path to the bbolt database]` - - Example: - - ```bash - $bbolt info ~/default.etcd/member/snap/db - Page Size: 4096 - ``` - - - **note**: page size is given in bytes - - Bbolt database is using page size of 4KB - -### buckets - -- `buckets` print a list of buckets of Bbolt database is currently having. Find more information on buckets [here](https://github.com/etcd-io/bbolt#using-buckets) -- usage: - `bbolt buckets [path to the bbolt database]` - - Example: - - ```bash - $bbolt buckets ~/default.etcd/member/snap/db - alarm - auth - authRoles - authUsers - cluster - key - lease - members - members_removed - meta - ``` - - - It means when you start an etcd, it creates these `10` buckets using bbolt database. - -### check - -- `check` opens a database at a given `[PATH]` and runs an exhaustive check to verify that all pages are accessible or are marked as freed. It also verifies that no pages are double referenced. -- usage: - `bbolt check [path to the bbolt database]` - - Example: - - ```bash - $bbolt check ~/default.etcd/member/snap/db - ok - ``` - - - It returns `ok` as our database file `db` is not corrupted. - -### stats - -- To gather essential statistics about the bbolt database: `stats` performs an extensive search of the database to track every page reference. It starts at the current meta page and recursively iterates through every accessible bucket. -- usage: - `bbolt stats [path to the bbolt database]` - - Example: - - ```bash - $bbolt stats ~/default.etcd/member/snap/db - Aggregate statistics for 10 buckets - - Page count statistics - Number of logical branch pages: 0 - Number of physical branch overflow pages: 0 - Number of logical leaf pages: 0 - Number of physical leaf overflow pages: 0 - Tree statistics - Number of keys/value pairs: 11 - Number of levels in B+tree: 1 - Page size utilization - Bytes allocated for physical branch pages: 0 - Bytes actually used for branch data: 0 (0%) - Bytes allocated for physical leaf pages: 0 - Bytes actually used for leaf data: 0 (0%) - Bucket statistics - Total number of buckets: 10 - Total number on inlined buckets: 10 (100%) - Bytes used for inlined buckets: 780 (0%) - ``` - -### inspect -- `inspect` inspect the structure of the database. -- Usage: `bbolt inspect [path to the bbolt database]` - - Example: -```bash -$ ./bbolt inspect ~/default.etcd/member/snap/db -{ - "name": "root", - "keyN": 0, - "buckets": [ - { - "name": "alarm", - "keyN": 0 - }, - { - "name": "auth", - "keyN": 2 - }, - { - "name": "authRoles", - "keyN": 1 - }, - { - "name": "authUsers", - "keyN": 1 - }, - { - "name": "cluster", - "keyN": 1 - }, - { - "name": "key", - "keyN": 1285 - }, - { - "name": "lease", - "keyN": 2 - }, - { - "name": "members", - "keyN": 1 - }, - { - "name": "members_removed", - "keyN": 0 - }, - { - "name": "meta", - "keyN": 3 - } - ] -} -``` - -### pages - -- Pages prints a table of pages with their type (meta, leaf, branch, freelist). -- The `meta` will store the metadata information of database. -- The `leaf` and `branch` pages will show a key count in the `items` column. -- The `freelist` will show the number of free pages, which are free for writing again. -- The `overflow` column shows the number of blocks that the page spills over into. -- usage: - `bbolt pages [path to the bbolt database]` - - Example: - - ```bash - $bbolt pages ~/default.etcd/member/snap/db - ID TYPE ITEMS OVRFLW - ======== ========== ====== ====== - 0 meta 0 - 1 meta 0 - 2 free - 3 leaf 10 - 4 freelist 2 - 5 free - ``` - -### page - -- Page prints one or more pages in human readable format. -- usage: - - ```bash - bolt page [path to the bbolt database] pageid [pageid...] - or: bolt page --all [path to the bbolt database] - - Additional options include: - - --all - prints all pages (only skips pages that were considered successful overflow pages) - --format-value=auto|ascii-encoded|hex|bytes|redacted (default: auto) - prints values (on the leaf page) using the given format - ``` - - Example: - - ```bash - $bbolt page ~/default.etcd/member/snap/db 3 - Page ID: 3 - Page Type: leaf - Total Size: 4096 bytes - Overflow pages: 0 - Item Count: 10 - - "alarm": - "auth": - "authRoles": - "authUsers": - "cluster": - "key": - "lease": - "members": - "members_removed": - "meta": - ``` - - - It prints information of page `page ID: 3` - -### page-item - -- page-item prints a page item's key and value. -- usage: - - ```bash - bolt page-item [options] [path to the bbolt database] - Additional options include: - - --key-only - Print only the key - --value-only - Print only the value - --format - Output format. One of: auto|ascii-encoded|hex|bytes|redacted (default=auto) - ``` - - Example: - - ```bash - $bbolt page-item --key-only ~/default.etcd/member/snap/db 3 7 - "members" - ``` - - - It returns the key as `--key-only` flag is passed of `pageID: 3` and `itemID: 7` - -### dump - -- Dump prints a hexadecimal dump of one or more given pages. -- usage: - `bolt dump [path to the bbolt database] [pageid...]` - -### keys - -- Print a list of keys in the given bucket. -- usage: - - ```bash - bolt keys [path to the bbolt database] [BucketName] - - Additional options include: - --format - Output format. One of: auto|ascii-encoded|hex|bytes|redacted (default=auto) - ``` - - Example 1: - - ```bash - $bbolt keys ~/default.etcd/member/snap/db meta - confState - consistent_index - term - ``` - - - It list all the keys in bucket: `meta` - - Example 2: - - ```bash - $bbolt keys ~/default.etcd/member/snap/db members - 8e9e05c52164694d - ``` - - - It list all the keys in `members` bucket which is a `memberId` of etcd cluster member. - - In this case we are running a single member etcd cluster, hence only `one memberId` is present. If we would have run a `3` member etcd cluster then it will return a `3 memberId` as `3 cluster members` would have been present in `members` bucket. - -### get - -- Print the value of the given key in the given bucket. -- usage: - - ```bash - bolt get [path to the bbolt database] [BucketName] [Key] - - Additional options include: - --format - Output format. One of: auto|ascii-encoded|hex|bytes|redacted (default=auto) - --parse-format - Input format (of key). One of: ascii-encoded|hex (default=ascii-encoded)" - ``` - - Example 1: - - ```bash - $bbolt get --format=hex ~/default.etcd/member/snap/db meta term - 0000000000000004 - ``` - - - It returns the value present in bucket: `meta` for key: `term` in hexadecimal format. - - Example 2: - - ```bash - $bbolt get ~/default.etcd/member/snap/db members 8e9e05c52164694d - {"id":10276657743932975437,"peerURLs":["http://localhost:2380"],"name":"default","clientURLs":["http://localhost:2379"]} - ``` - - - It returns the value present in bucket: `members` for key: `8e9e05c52164694d`. - -### compact - -- Compact opens a database at given `[Source Path]` and walks it recursively, copying keys as they are found from all buckets, to a newly created database at `[Destination Path]`. The original database is left untouched. -- usage: - - ```bash - bbolt compact [options] -o [Destination Path] [Source Path] - - Additional options include: - - -tx-max-size NUM - Specifies the maximum size of individual transactions. - Defaults to 64KB - ``` - - Example: - - ```bash - $bbolt compact -o ~/db.compact ~/default.etcd/member/snap/db - 16805888 -> 32768 bytes (gain=512.88x) - ``` - - - It will create a compacted database file: `db.compact` at given path. - -### bench - -- run synthetic benchmark against bbolt database. -- usage: - - ```bash - Usage: - -batch-size int - - -blockprofile string - - -count int - (default 1000) - -cpuprofile string - - -fill-percent float - (default 0.5) - -key-size int - (default 8) - -memprofile string - - -no-sync - - -path string - - -profile-mode string - (default "rw") - -read-mode string - (default "seq") - -value-size int - (default 32) - -work - - -write-mode string - (default "seq") - ``` - - Example: - - ```bash - $bbolt bench ~/default.etcd/member/snap/db -batch-size 400 -key-size 16 - # Write 68.523572ms (68.523µs/op) (14593 op/sec) - # Read 1.000015152s (11ns/op) (90909090 op/sec) - ``` - - - It runs a benchmark with batch size of `400` and with key size of `16` while for others parameters default value is taken. +# Introduction to bbolt command line + +`bbolt` provides a command line utility for inspecting and manipulating bbolt database files. To install bbolt command-line please refer [here](https://github.com/etcd-io/bbolt#installing) + +**Note**: [etcd](https://github.com/etcd-io/etcd) uses bbolt as its backend storage engine. In this document, we take etcd as an example to demonstrate the usage of bbolt commands. Refer to [install etcd](https://etcd.io/docs/v3.5/install/) for installing etcd. + +1. Start a single member etcd cluster with this command below: + + ```bash + $etcd + ``` + + It will create a directory `default.etcd` by default under current working directory, and the directory structure will look like this: + + ```bash + $tree default.etcd + default.etcd + └── member + ├── snap + │   └── db // this is bbolt database file + └── wal + └── 0000000000000000-0000000000000000.wal + + 3 directories, 2 files + ``` + +2. Put some dummy data using [etcdctl](https://github.com/etcd-io/etcd/tree/main/etcdctl). +3. Stop the etcd instance. Note a bbolt database file can only be opened by one read-write process, because it is exclusively locked when opened. + +## Usage + +- `bbolt command [arguments]` + +### help + +- help will print information about that command + + ```bash + $bbolt help + + The commands are: + + version prints the current version of bbolt + bench run synthetic benchmark against bbolt + buckets print a list of buckets + check verifies integrity of bbolt database + compact copies a bbolt database, compacting it in the process + dump print a hexadecimal dump of a single page + get print the value of a key in a bucket + info print basic info + keys print a list of keys in a bucket + help print this screen + page print one or more pages in human readable format + pages print list of pages with their types + page-item print the key and value of a page item. + stats iterate over all pages and generate usage stats + surgery perform surgery on bbolt database + ``` + +- you can use `help` with any command: `bbolt [command] -h` for more information about command. + +## Analyse bbolt database with bbolt command line + +### version + +- `version` print the current version information of bbolt command-line. +- usage: + `bbolt version` + + Example: + + ```bash + $bbolt version + bbolt version: 1.3.7 + Go Version: go1.21.6 + Go OS/Arch: darwin/arm64 + ``` + +### info + +- `info` print the basic information about the given Bbolt database. +- usage: + `bbolt info [path to the bbolt database]` + + Example: + + ```bash + $bbolt info ~/default.etcd/member/snap/db + Page Size: 4096 + ``` + + - **note**: page size is given in bytes + - Bbolt database is using page size of 4KB + +### buckets + +- `buckets` print a list of buckets of Bbolt database is currently having. Find more information on buckets [here](https://github.com/etcd-io/bbolt#using-buckets) +- usage: + `bbolt buckets [path to the bbolt database]` + + Example: + + ```bash + $bbolt buckets ~/default.etcd/member/snap/db + alarm + auth + authRoles + authUsers + cluster + key + lease + members + members_removed + meta + ``` + + - It means when you start an etcd, it creates these `10` buckets using bbolt database. + +### check + +- `check` opens a database at a given `[PATH]` and runs an exhaustive check to verify that all pages are accessible or are marked as freed. It also verifies that no pages are double referenced. +- usage: + `bbolt check [path to the bbolt database]` + + Example: + + ```bash + $bbolt check ~/default.etcd/member/snap/db + ok + ``` + + - It returns `ok` as our database file `db` is not corrupted. + +### stats + +- To gather essential statistics about the bbolt database: `stats` performs an extensive search of the database to track every page reference. It starts at the current meta page and recursively iterates through every accessible bucket. +- usage: + `bbolt stats [path to the bbolt database]` + + Example: + + ```bash + $bbolt stats ~/default.etcd/member/snap/db + Aggregate statistics for 10 buckets + + Page count statistics + Number of logical branch pages: 0 + Number of physical branch overflow pages: 0 + Number of logical leaf pages: 0 + Number of physical leaf overflow pages: 0 + Tree statistics + Number of keys/value pairs: 11 + Number of levels in B+tree: 1 + Page size utilization + Bytes allocated for physical branch pages: 0 + Bytes actually used for branch data: 0 (0%) + Bytes allocated for physical leaf pages: 0 + Bytes actually used for leaf data: 0 (0%) + Bucket statistics + Total number of buckets: 10 + Total number on inlined buckets: 10 (100%) + Bytes used for inlined buckets: 780 (0%) + ``` + +### inspect +- `inspect` inspect the structure of the database. +- Usage: `bbolt inspect [path to the bbolt database]` + + Example: +```bash +$ ./bbolt inspect ~/default.etcd/member/snap/db +{ + "name": "root", + "keyN": 0, + "buckets": [ + { + "name": "alarm", + "keyN": 0 + }, + { + "name": "auth", + "keyN": 2 + }, + { + "name": "authRoles", + "keyN": 1 + }, + { + "name": "authUsers", + "keyN": 1 + }, + { + "name": "cluster", + "keyN": 1 + }, + { + "name": "key", + "keyN": 1285 + }, + { + "name": "lease", + "keyN": 2 + }, + { + "name": "members", + "keyN": 1 + }, + { + "name": "members_removed", + "keyN": 0 + }, + { + "name": "meta", + "keyN": 3 + } + ] +} +``` + +### pages + +- Pages prints a table of pages with their type (meta, leaf, branch, freelist). +- The `meta` will store the metadata information of database. +- The `leaf` and `branch` pages will show a key count in the `items` column. +- The `freelist` will show the number of free pages, which are free for writing again. +- The `overflow` column shows the number of blocks that the page spills over into. +- usage: + `bbolt pages [path to the bbolt database]` + + Example: + + ```bash + $bbolt pages ~/default.etcd/member/snap/db + ID TYPE ITEMS OVRFLW + ======== ========== ====== ====== + 0 meta 0 + 1 meta 0 + 2 free + 3 leaf 10 + 4 freelist 2 + 5 free + ``` + +### page + +- Page prints one or more pages in human readable format. +- usage: + + ```bash + bolt page [path to the bbolt database] pageid [pageid...] + or: bolt page --all [path to the bbolt database] + + Additional options include: + + --all + prints all pages (only skips pages that were considered successful overflow pages) + --format-value=auto|ascii-encoded|hex|bytes|redacted (default: auto) + prints values (on the leaf page) using the given format + ``` + + Example: + + ```bash + $bbolt page ~/default.etcd/member/snap/db 3 + Page ID: 3 + Page Type: leaf + Total Size: 4096 bytes + Overflow pages: 0 + Item Count: 10 + + "alarm": + "auth": + "authRoles": + "authUsers": + "cluster": + "key": + "lease": + "members": + "members_removed": + "meta": + ``` + + - It prints information of page `page ID: 3` + +### page-item + +- page-item prints a page item's key and value. +- usage: + + ```bash + bolt page-item [options] [path to the bbolt database] + Additional options include: + + --key-only + Print only the key + --value-only + Print only the value + --format + Output format. One of: auto|ascii-encoded|hex|bytes|redacted (default=auto) + ``` + + Example: + + ```bash + $bbolt page-item --key-only ~/default.etcd/member/snap/db 3 7 + "members" + ``` + + - It returns the key as `--key-only` flag is passed of `pageID: 3` and `itemID: 7` + +### dump + +- Dump prints a hexadecimal dump of one or more given pages. +- usage: + `bolt dump [path to the bbolt database] [pageid...]` + +### keys + +- Print a list of keys in the given bucket. +- usage: + + ```bash + bolt keys [path to the bbolt database] [BucketName] + + Additional options include: + --format + Output format. One of: auto|ascii-encoded|hex|bytes|redacted (default=auto) + ``` + + Example 1: + + ```bash + $bbolt keys ~/default.etcd/member/snap/db meta + confState + consistent_index + term + ``` + + - It list all the keys in bucket: `meta` + + Example 2: + + ```bash + $bbolt keys ~/default.etcd/member/snap/db members + 8e9e05c52164694d + ``` + + - It list all the keys in `members` bucket which is a `memberId` of etcd cluster member. + - In this case we are running a single member etcd cluster, hence only `one memberId` is present. If we would have run a `3` member etcd cluster then it will return a `3 memberId` as `3 cluster members` would have been present in `members` bucket. + +### get + +- Print the value of the given key in the given bucket. +- usage: + + ```bash + bolt get [path to the bbolt database] [BucketName] [Key] + + Additional options include: + --format + Output format. One of: auto|ascii-encoded|hex|bytes|redacted (default=auto) + --parse-format + Input format (of key). One of: ascii-encoded|hex (default=ascii-encoded)" + ``` + + Example 1: + + ```bash + $bbolt get --format=hex ~/default.etcd/member/snap/db meta term + 0000000000000004 + ``` + + - It returns the value present in bucket: `meta` for key: `term` in hexadecimal format. + + Example 2: + + ```bash + $bbolt get ~/default.etcd/member/snap/db members 8e9e05c52164694d + {"id":10276657743932975437,"peerURLs":["http://localhost:2380"],"name":"default","clientURLs":["http://localhost:2379"]} + ``` + + - It returns the value present in bucket: `members` for key: `8e9e05c52164694d`. + +### compact + +- Compact opens a database at given `[Source Path]` and walks it recursively, copying keys as they are found from all buckets, to a newly created database at `[Destination Path]`. The original database is left untouched. +- usage: + + ```bash + bbolt compact [options] -o [Destination Path] [Source Path] + + Additional options include: + + -tx-max-size NUM + Specifies the maximum size of individual transactions. + Defaults to 64KB + ``` + + Example: + + ```bash + $bbolt compact -o ~/db.compact ~/default.etcd/member/snap/db + 16805888 -> 32768 bytes (gain=512.88x) + ``` + + - It will create a compacted database file: `db.compact` at given path. + +### bench + +- run synthetic benchmark against bbolt database. +- usage: + + ```bash + Usage: + -batch-size int + + -blockprofile string + + -count int + (default 1000) + -cpuprofile string + + -fill-percent float + (default 0.5) + -key-size int + (default 8) + -memprofile string + + -no-sync + + -path string + + -profile-mode string + (default "rw") + -read-mode string + (default "seq") + -value-size int + (default 32) + -work + + -write-mode string + (default "seq") + ``` + + Example: + + ```bash + $bbolt bench ~/default.etcd/member/snap/db -batch-size 400 -key-size 16 + # Write 68.523572ms (68.523µs/op) (14593 op/sec) + # Read 1.000015152s (11ns/op) (90909090 op/sec) + ``` + + - It runs a benchmark with batch size of `400` and with key size of `16` while for others parameters default value is taken. diff --git a/code-of-conduct.md b/code-of-conduct.md index f78dd84..a7b88a7 100644 --- a/code-of-conduct.md +++ b/code-of-conduct.md @@ -1,3 +1,3 @@ -# etcd Community Code of Conduct - -Please refer to [etcd Community Code of Conduct](https://github.com/etcd-io/etcd/blob/main/code-of-conduct.md). +# etcd Community Code of Conduct + +Please refer to [etcd Community Code of Conduct](https://github.com/etcd-io/etcd/blob/main/code-of-conduct.md). diff --git a/go.sum b/go.sum index de26c3e..24a60c5 100644 --- a/go.sum +++ b/go.sum @@ -1,24 +1,24 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= -go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= +go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scripts/compare_benchmarks.sh b/scripts/compare_benchmarks.sh index 2b77669..0cd4a24 100644 --- a/scripts/compare_benchmarks.sh +++ b/scripts/compare_benchmarks.sh @@ -1,70 +1,70 @@ -#!/usr/bin/env bash -# https://github.com/kubernetes/kube-state-metrics/blob/main/tests/compare_benchmarks.sh (originally written by mxinden) - -# exit immediately when a command fails -set -e -# only exit with zero if all commands of the pipeline exit successfully -set -o pipefail -# error on unset variables -set -u - -[[ "$#" -eq 1 ]] || echo "One argument required, $# provided." - -REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)" -BASE_TO_COMPARE=$1 - -RESULT_CURRENT="$(mktemp)-${REF_CURRENT}" -RESULT_TO_COMPARE="$(mktemp)-${BASE_TO_COMPARE}" - -BENCH_COUNT=${BENCH_COUNT:-10} -BENCHSTAT_CONFIDENCE_LEVEL=${BENCHSTAT_CONFIDENCE_LEVEL:-0.9} -BENCHSTAT_FORMAT=${BENCHSTAT_FORMAT:-"text"} -BENCH_PARAMETERS=${BENCH_PARAMETERS:-"-count 2000000 -batch-size 10000"} - -if [[ "${BENCHSTAT_FORMAT}" == "csv" ]] && [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then - echo "BENCHSTAT_FORMAT is set to csv, but BENCHSTAT_OUTPUT_FILE is not set." - exit 1 -fi - -function bench() { - local output_file - output_file="$1" - make build - - for _ in $(seq "$BENCH_COUNT"); do - echo ./bin/bbolt bench -gobench-output -profile-mode n ${BENCH_PARAMETERS} - # shellcheck disable=SC2086 - ./bin/bbolt bench -gobench-output -profile-mode n ${BENCH_PARAMETERS} >> "${output_file}" - done -} - -function main() { - echo "### Benchmarking PR ${REF_CURRENT}" - bench "${RESULT_CURRENT}" - echo "" - echo "### Done benchmarking ${REF_CURRENT}" - - echo "### Benchmarking base ${BASE_TO_COMPARE}" - git checkout "${BASE_TO_COMPARE}" - bench "${RESULT_TO_COMPARE}" - echo "" - echo "### Done benchmarking ${BASE_TO_COMPARE}" - - git checkout - - - echo "" - echo "### Result" - echo "BASE=${BASE_TO_COMPARE} HEAD=${REF_CURRENT}" - - if [[ "${BENCHSTAT_FORMAT}" == "csv" ]]; then - benchstat -format=csv -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 2>/dev/null 1>"${BENCHSTAT_OUTPUT_FILE}" - else - if [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then - benchstat -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" - else - benchstat -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 1>"${BENCHSTAT_OUTPUT_FILE}" - fi - fi -} - -main +#!/usr/bin/env bash +# https://github.com/kubernetes/kube-state-metrics/blob/main/tests/compare_benchmarks.sh (originally written by mxinden) + +# exit immediately when a command fails +set -e +# only exit with zero if all commands of the pipeline exit successfully +set -o pipefail +# error on unset variables +set -u + +[[ "$#" -eq 1 ]] || echo "One argument required, $# provided." + +REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)" +BASE_TO_COMPARE=$1 + +RESULT_CURRENT="$(mktemp)-${REF_CURRENT}" +RESULT_TO_COMPARE="$(mktemp)-${BASE_TO_COMPARE}" + +BENCH_COUNT=${BENCH_COUNT:-10} +BENCHSTAT_CONFIDENCE_LEVEL=${BENCHSTAT_CONFIDENCE_LEVEL:-0.9} +BENCHSTAT_FORMAT=${BENCHSTAT_FORMAT:-"text"} +BENCH_PARAMETERS=${BENCH_PARAMETERS:-"-count 2000000 -batch-size 10000"} + +if [[ "${BENCHSTAT_FORMAT}" == "csv" ]] && [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then + echo "BENCHSTAT_FORMAT is set to csv, but BENCHSTAT_OUTPUT_FILE is not set." + exit 1 +fi + +function bench() { + local output_file + output_file="$1" + make build + + for _ in $(seq "$BENCH_COUNT"); do + echo ./bin/bbolt bench -gobench-output -profile-mode n ${BENCH_PARAMETERS} + # shellcheck disable=SC2086 + ./bin/bbolt bench -gobench-output -profile-mode n ${BENCH_PARAMETERS} >> "${output_file}" + done +} + +function main() { + echo "### Benchmarking PR ${REF_CURRENT}" + bench "${RESULT_CURRENT}" + echo "" + echo "### Done benchmarking ${REF_CURRENT}" + + echo "### Benchmarking base ${BASE_TO_COMPARE}" + git checkout "${BASE_TO_COMPARE}" + bench "${RESULT_TO_COMPARE}" + echo "" + echo "### Done benchmarking ${BASE_TO_COMPARE}" + + git checkout - + + echo "" + echo "### Result" + echo "BASE=${BASE_TO_COMPARE} HEAD=${REF_CURRENT}" + + if [[ "${BENCHSTAT_FORMAT}" == "csv" ]]; then + benchstat -format=csv -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 2>/dev/null 1>"${BENCHSTAT_OUTPUT_FILE}" + else + if [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then + benchstat -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" + else + benchstat -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 1>"${BENCHSTAT_OUTPUT_FILE}" + fi + fi +} + +main diff --git a/scripts/fix.sh b/scripts/fix.sh index 6b933c9..34dcc34 100644 --- a/scripts/fix.sh +++ b/scripts/fix.sh @@ -1,13 +1,13 @@ -GO_CMD="go" - -# TODO(ptabor): Expand to cover different architectures (GOOS GOARCH), or just list go files. - -GOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .GoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) -TESTGOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .TestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) -XTESTGOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .XTestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) - - -echo "${GOFILES}" "${TESTGOFILES}" "${XTESTGOFILES}"| xargs -n 100 go run golang.org/x/tools/cmd/goimports@latest -w -local go.etcd.io - -go fmt ./... -go mod tidy +GO_CMD="go" + +# TODO(ptabor): Expand to cover different architectures (GOOS GOARCH), or just list go files. + +GOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .GoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) +TESTGOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .TestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) +XTESTGOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .XTestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) + + +echo "${GOFILES}" "${TESTGOFILES}" "${XTESTGOFILES}"| xargs -n 100 go run golang.org/x/tools/cmd/goimports@latest -w -local go.etcd.io + +go fmt ./... +go mod tidy