mirror of
https://github.com/VibedByKaKi/forgejo-11-secure.git
synced 2026-10-15 06:48:03 +02:00
28dbbf44cf
**Backport:** #10686 (cherry picked from commit 270317a3ad28562f84b0fe65c0a32a69d74e3a3d) ``` NAME: forgejo doctor cleanup-commit-status - Cleanup extra records in commit_status table USAGE: forgejo doctor cleanup-commit-status DESCRIPTION: Forgejo suffered from a bug which caused the creation of more entries in the "commit_status" table than necessary. This operation removes the redundant data caused by the bug. Removing this data is almost always safe. These reundant records can be accessed by users through the API, making it possible, but unlikely, that removing it could have an impact to integrating services (API: /repos/{owner}/{repo}/commits/{ref}/statuses). It is safe to run while Forgejo is online. On very large Forgejo instances, the performance of operation will improve if the buffer-size option is used with large values. Approximately 130 MB of memory is required for every 100,000 records in the buffer. Bug reference: https://codeberg.org/forgejo/forgejo/issues/10671 OPTIONS: --help, -h show help --custom-path string, -C string Set custom path (defaults to '{WorkPath}/custom') --config string, -c string Set custom config file (defaults to '{WorkPath}/custom/conf/app.ini') --work-path string, -w string Set Forgejo's working path (defaults to the directory of the Forgejo binary) --verbose, -V Show process details --dry-run Report statistics from the operation but do not modify the database --buffer-size int Record count per query while iterating records; larger values are typically faster but use more memory (default: 100000) --delete-chunk-size int Number of records to delete per DELETE query (default: 1000) ``` The cleanup effectively performs `SELECT * FROM commit_status ORDER BY repo_id, sha, context, index, id`, and iterates through the records. Whenever `index, id` changes without the other fields changing, then it's a useless record that can be deleted. The major complication is doing that at scale without bringing the entire database table into memory, which is performed through a new iteration method `IterateByKeyset`. Manually tested against a 455,303 record table in PostgreSQL, MySQL, and SQLite, which was reduced to 10,781 records, dropping 97.5% of the records. Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/10783 Reviewed-by: Gusted <gusted@noreply.codeberg.org>
79 lines
2.3 KiB
Go
79 lines
2.3 KiB
Go
// Copyright 2022 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package db_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"testing"
|
|
|
|
"forgejo.org/models/db"
|
|
git_model "forgejo.org/models/git"
|
|
repo_model "forgejo.org/models/repo"
|
|
"forgejo.org/models/unittest"
|
|
"forgejo.org/modules/setting"
|
|
"forgejo.org/modules/test"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestIterate(t *testing.T) {
|
|
require.NoError(t, unittest.PrepareTestDatabase())
|
|
xe := unittest.GetXORMEngine()
|
|
require.NoError(t, xe.Sync(&repo_model.RepoUnit{}))
|
|
defer test.MockVariableValue(&setting.Database.IterateBufferSize, 50)()
|
|
|
|
cnt, err := db.GetEngine(db.DefaultContext).Count(&repo_model.RepoUnit{})
|
|
require.NoError(t, err)
|
|
|
|
var repoUnitCnt int
|
|
err = db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repo *repo_model.RepoUnit) error {
|
|
repoUnitCnt++
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
assert.EqualValues(t, cnt, repoUnitCnt)
|
|
|
|
err = db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repoUnit *repo_model.RepoUnit) error {
|
|
has, err := db.ExistByID[repo_model.RepoUnit](ctx, repoUnit.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !has {
|
|
return db.ErrNotExist{Resource: "repo_unit", ID: repoUnit.ID}
|
|
}
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestIterateMultipleFields(t *testing.T) {
|
|
for _, bufferSize := range []int{1, 2, 3, 10} { // 8 records in fixture
|
|
t.Run(fmt.Sprintf("No Modifications bufferSize=%d", bufferSize), func(t *testing.T) {
|
|
require.NoError(t, unittest.PrepareTestDatabase())
|
|
|
|
// Fetch all the commit status IDs...
|
|
var remainingIDs []int64
|
|
err := db.GetEngine(t.Context()).Table(&git_model.CommitStatus{}).Cols("id").Find(&remainingIDs)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, remainingIDs)
|
|
|
|
// Ensure that every repo unit ID is found when doing iterate:
|
|
err = db.IterateByKeyset(t.Context(),
|
|
nil,
|
|
[]string{"repo_id", "sha", "context", "index", "id"},
|
|
bufferSize,
|
|
func(ctx context.Context, commit_status *git_model.CommitStatus) error {
|
|
remainingIDs = slices.DeleteFunc(remainingIDs, func(n int64) bool {
|
|
return commit_status.ID == n
|
|
})
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Empty(t, remainingIDs)
|
|
})
|
|
}
|
|
}
|