Skip to content

Commit da5975b

Browse files
ahrtrahrtr
authored
Merge pull request #659 from Elbehery/add_test_check_page
Enhance check functionality to support checking starting from a pageId
2 parents 1906d5a + 29d1e3d commit da5975b

File tree

2 files changed

+174
-9
lines changed

2 files changed

+174
-9
lines changed

‎tx_check.go

+65-9
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,63 @@ func (tx *Tx) check(cfg checkConfig, ch chan error) {
6060
}
6161
}
6262

63-
// Recursively check buckets.
64-
tx.recursivelyCheckBucket(&tx.root, reachable, freed, cfg.kvStringer, ch)
65-
66-
// Ensure all pages below high water mark are either reachable or freed.
67-
for i := common.Pgid(0); i < tx.meta.Pgid(); i++ {
68-
_, isReachable := reachable[i]
69-
if !isReachable && !freed[i] {
70-
ch <- fmt.Errorf("page %d: unreachable unfreed", int(i))
63+
if cfg.pageId == 0 {
64+
// Check the whole db file, starting from the root bucket and
65+
// recursively check all child buckets.
66+
tx.recursivelyCheckBucket(&tx.root, reachable, freed, cfg.kvStringer, ch)
67+
68+
// Ensure all pages below high water mark are either reachable or freed.
69+
for i := common.Pgid(0); i < tx.meta.Pgid(); i++ {
70+
_, isReachable := reachable[i]
71+
if !isReachable && !freed[i] {
72+
ch <- fmt.Errorf("page %d: unreachable unfreed", int(i))
73+
}
74+
}
75+
} else {
76+
// Check the db file starting from a specified pageId.
77+
if cfg.pageId < 2 || cfg.pageId >= uint(tx.meta.Pgid()) {
78+
ch <- fmt.Errorf("page ID (%d) out of range [%d, %d)", cfg.pageId, 2, tx.meta.Pgid())
79+
return
80+
}
81+
82+
tx.recursivelyCheckPage(common.Pgid(cfg.pageId), reachable, freed, cfg.kvStringer, ch)
83+
}
84+
}
85+
86+
func (tx *Tx) recursivelyCheckPage(pageId common.Pgid, reachable map[common.Pgid]*common.Page, freed map[common.Pgid]bool,
87+
kvStringer KVStringer, ch chan error) {
88+
tx.checkInvariantProperties(pageId, reachable, freed, kvStringer, ch)
89+
tx.recursivelyCheckBucketInPage(pageId, reachable, freed, kvStringer, ch)
90+
}
91+
92+
func (tx *Tx) recursivelyCheckBucketInPage(pageId common.Pgid, reachable map[common.Pgid]*common.Page, freed map[common.Pgid]bool,
93+
kvStringer KVStringer, ch chan error) {
94+
p := tx.page(pageId)
95+
96+
switch {
97+
case p.IsBranchPage():
98+
for i := range p.BranchPageElements() {
99+
elem := p.BranchPageElement(uint16(i))
100+
tx.recursivelyCheckBucketInPage(elem.Pgid(), reachable, freed, kvStringer, ch)
101+
}
102+
case p.IsLeafPage():
103+
for i := range p.LeafPageElements() {
104+
elem := p.LeafPageElement(uint16(i))
105+
if elem.Flags()&common.BucketLeafFlag != 0 {
106+
107+
inBkt := common.NewInBucket(pageId, 0)
108+
tmpBucket := Bucket{
109+
InBucket: &inBkt,
110+
rootNode: &node{isLeaf: p.IsLeafPage()},
111+
FillPercent: DefaultFillPercent,
112+
}
113+
if child := tmpBucket.Bucket(elem.Key()); child != nil {
114+
tx.recursivelyCheckBucket(&tmpBucket, reachable, freed, kvStringer, ch)
115+
}
116+
}
71117
}
118+
default:
119+
ch <- fmt.Errorf("unexpected page type (flags: %x) for pgId:%d", p.Flags(), pageId)
72120
}
73121
}
74122

@@ -167,7 +215,7 @@ func (tx *Tx) recursivelyCheckPageKeyOrderInternal(
167215
return p.LeafPageElement(p.Count() - 1).Key()
168216
}
169217
default:
170-
ch <- fmt.Errorf("unexpected page type for pgId:%d", pgId)
218+
ch <- fmt.Errorf("unexpected page type (flags: %x) for pgId:%d", p.Flags(), pgId)
171219
}
172220
return maxKeyInSubtree
173221
}
@@ -202,6 +250,7 @@ func verifyKeyOrder(pgId common.Pgid, pageType string, index int, key []byte, pr
202250

203251
type checkConfig struct {
204252
kvStringer KVStringer
253+
pageId uint
205254
}
206255

207256
type CheckOption func(options *checkConfig)
@@ -212,6 +261,13 @@ func WithKVStringer(kvStringer KVStringer) CheckOption {
212261
}
213262
}
214263

264+
// WithPageId sets a page ID from which the check command starts to check
265+
func WithPageId(pageId uint) CheckOption {
266+
return func(c *checkConfig) {
267+
c.pageId = pageId
268+
}
269+
}
270+
215271
// KVStringer allows to prepare human-readable diagnostic messages.
216272
type KVStringer interface {
217273
KeyToString([]byte) string

‎tx_check_test.go

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package bbolt_test
2+
3+
import (
4+
"fmt"
5+
"math/rand"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
10+
"go.etcd.io/bbolt"
11+
"go.etcd.io/bbolt/internal/btesting"
12+
"go.etcd.io/bbolt/internal/common"
13+
"go.etcd.io/bbolt/internal/guts_cli"
14+
)
15+
16+
func TestTx_Check_CorruptPage(t *testing.T) {
17+
t.Log("Creating db file.")
18+
db := btesting.MustCreateDBWithOption(t, &bbolt.Options{PageSize: 4096})
19+
20+
// Each page can hold roughly 20 key/values pair, so 100 such
21+
// key/value pairs will consume about 5 leaf pages.
22+
err := db.Fill([]byte("data"), 1, 100,
23+
func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
24+
func(tx int, k int) []byte { return make([]byte, 100) },
25+
)
26+
require.NoError(t, err)
27+
28+
t.Log("Corrupting random leaf page.")
29+
victimPageId, validPageIds := corruptRandomLeafPage(t, db.DB)
30+
31+
t.Log("Running consistency check.")
32+
vErr := db.View(func(tx *bbolt.Tx) error {
33+
var cErrs []error
34+
35+
t.Log("Check corrupted page.")
36+
errChan := tx.Check(bbolt.WithPageId(uint(victimPageId)))
37+
for cErr := range errChan {
38+
cErrs = append(cErrs, cErr)
39+
}
40+
require.Greater(t, len(cErrs), 0)
41+
42+
t.Log("Check valid pages.")
43+
cErrs = cErrs[:0]
44+
for _, pgId := range validPageIds {
45+
errChan = tx.Check(bbolt.WithPageId(uint(pgId)))
46+
for cErr := range errChan {
47+
cErrs = append(cErrs, cErr)
48+
}
49+
require.Equal(t, 0, len(cErrs))
50+
}
51+
return nil
52+
})
53+
require.NoError(t, vErr)
54+
t.Log("All check passed")
55+
56+
// Manually close the db, otherwise the PostTestCleanup will
57+
// check the db again and accordingly fail the test.
58+
db.MustClose()
59+
}
60+
61+
// corruptRandomLeafPage corrupts one random leaf page.
62+
func corruptRandomLeafPage(t testing.TB, db *bbolt.DB) (victimPageId common.Pgid, validPageIds []common.Pgid) {
63+
victimPageId, validPageIds = pickupRandomLeafPage(t, db)
64+
victimPage, victimBuf, err := guts_cli.ReadPage(db.Path(), uint64(victimPageId))
65+
require.NoError(t, err)
66+
require.True(t, victimPage.IsLeafPage())
67+
require.True(t, victimPage.Count() > 1)
68+
69+
// intentionally make the second key < the first key.
70+
element := victimPage.LeafPageElement(1)
71+
key := element.Key()
72+
key[0] = 0
73+
74+
// Write the corrupt page to db file.
75+
err = guts_cli.WritePage(db.Path(), victimBuf)
76+
require.NoError(t, err)
77+
return victimPageId, validPageIds
78+
}
79+
80+
// pickupRandomLeafPage picks up a random leaf page.
81+
func pickupRandomLeafPage(t testing.TB, db *bbolt.DB) (victimPageId common.Pgid, validPageIds []common.Pgid) {
82+
// Read DB's RootPage, which should be a leaf page.
83+
rootPageId, _, err := guts_cli.GetRootPage(db.Path())
84+
require.NoError(t, err)
85+
rootPage, _, err := guts_cli.ReadPage(db.Path(), uint64(rootPageId))
86+
require.NoError(t, err)
87+
require.True(t, rootPage.IsLeafPage())
88+
89+
// The leaf page contains only one item, namely the bucket
90+
require.Equal(t, uint16(1), rootPage.Count())
91+
lpe := rootPage.LeafPageElement(uint16(0))
92+
require.True(t, lpe.IsBucketEntry())
93+
94+
// The bucket should be pointing to a branch page
95+
bucketRootPageId := lpe.Bucket().RootPage()
96+
bucketRootPage, _, err := guts_cli.ReadPage(db.Path(), uint64(bucketRootPageId))
97+
require.NoError(t, err)
98+
require.True(t, bucketRootPage.IsBranchPage())
99+
100+
// Retrieve all the leaf pages included in the branch page, and pick up random one from them.
101+
var bucketPageIds []common.Pgid
102+
for _, bpe := range bucketRootPage.BranchPageElements() {
103+
bucketPageIds = append(bucketPageIds, bpe.Pgid())
104+
}
105+
randomIdx := rand.Intn(len(bucketPageIds))
106+
victimPageId = bucketPageIds[randomIdx]
107+
validPageIds = append(bucketPageIds[:randomIdx], bucketPageIds[randomIdx+1:]...)
108+
return victimPageId, validPageIds
109+
}

0 commit comments

Comments
 (0)