Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disable generation of snapshots v1 #827

Merged
merged 2 commits into from
Oct 19, 2021
Merged

Disable generation of snapshots v1 #827

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1eacf5d
Disable generation of snapshots v1
sidenaio Oct 19, 2021
6f258c8
generate protobuf models
sidenaio Oct 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions blockchain/blockchain.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func (chain *Blockchain) loadPredefinedGenesis(network types.Network) (*types.Bl
}
defer stateDbData.Close()

if err = chain.appState.State.RecoverSnapshot(header.Height(), header.Root(), stateDbData); err != nil {
if err = chain.appState.State.RecoverSnapshot2(header.Height(), header.Root(), stateDbData); err != nil {
return nil, err
}

Expand All @@ -269,7 +269,7 @@ func (chain *Blockchain) loadPredefinedGenesis(network types.Network) (*types.Bl
}
defer identityStateDbData.Close()

if err = chain.appState.IdentityState.RecoverSnapshot(header.Height(), header.IdentityRoot(), identityStateDbData); err != nil {
if err = chain.appState.IdentityState.RecoverSnapshot2(header.Height(), header.IdentityRoot(), identityStateDbData); err != nil {
return nil, err
}

Expand Down Expand Up @@ -2384,12 +2384,11 @@ func (chain *Blockchain) GetIdentityDiff(height uint64) *state.IdentityStateDiff
}

func (chain *Blockchain) ReadSnapshotManifest() *snapshot.Manifest {
cid, cidV2, root, height, _ := chain.repo.LastSnapshotManifest()
if cid == nil {
cidV2, root, height, _ := chain.repo.LastSnapshotManifest()
if cidV2 == nil {
return nil
}
return &snapshot.Manifest{
Cid: cid,
CidV2: cidV2,
Root: root,
Height: height,
Expand Down
4 changes: 2 additions & 2 deletions cmd/genext/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func main() {
return err
}

stateRoot, err := state.WriteTreeTo(stateDb, genesis, file)
stateRoot, err := state.WriteTreeTo2(stateDb, genesis, file)
file.Close()
if err != nil {
return err
Expand All @@ -91,7 +91,7 @@ func main() {
return err
}

identityRoot, err := state.WriteTreeTo(identityStateDb, genesis, file)
identityRoot, err := state.WriteTreeTo2(identityStateDb, genesis, file)
file.Close()
if err != nil {
return err
Expand Down
4 changes: 2 additions & 2 deletions core/state/identity_statedb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,9 +448,9 @@ func (s *IdentityStateDB) SetPredefinedIdentities(state *models.ProtoPredefinedS
}
}

func (s *IdentityStateDB) RecoverSnapshot(height uint64, treeRoot common.Hash, from io.Reader) error {
func (s *IdentityStateDB) RecoverSnapshot2(height uint64, treeRoot common.Hash, from io.Reader) error {
pdb := dbm.NewPrefixDB(s.original, IdentityStateDbKeys.buildDbPrefix(height))
return ReadTreeFrom(pdb, height, treeRoot, from)
return ReadTreeFrom2(pdb, height, treeRoot, from)
}

func (s *IdentityStateDB) CommitSnapshot(height uint64) (dropDb dbm.DB) {
Expand Down
55 changes: 15 additions & 40 deletions core/state/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ const (

type SnapshotVersion byte

const SnapshotVersionV1 = SnapshotVersion(1)
const SnapshotVersionV2 = SnapshotVersion(2)

var (
Expand Down Expand Up @@ -95,19 +94,10 @@ func (m *SnapshotManager) createShapshotForVersion(height uint64, version Snapsh
return nil, common.Hash{}, ""
}

switch version {
case SnapshotVersionV1:
if root, err = m.state.WriteSnapshot(height, file); err != nil {
file.Close()
m.log.Error("Cannot write snapshot to file", "err", err)
return nil, common.Hash{}, ""
}
case SnapshotVersionV2:
if root, err = m.state.WriteSnapshot2(height, file); err != nil {
file.Close()
m.log.Error("Cannot write snapshot to file", "err", err)
return nil, common.Hash{}, ""
}
if root, err = m.state.WriteSnapshot2(height, file); err != nil {
file.Close()
m.log.Error("Cannot write snapshot to file", "err", err)
return nil, common.Hash{}, ""
}

file.Close()
Expand All @@ -132,19 +122,16 @@ func (m *SnapshotManager) createShapshotForVersion(height uint64, version Snapsh

func (m *SnapshotManager) createSnapshot(height uint64) (root common.Hash) {

cidV1, root, filePath := m.createShapshotForVersion(height, SnapshotVersionV1)
cidV2, _, filePath2 := m.createShapshotForVersion(height, SnapshotVersionV2)

if cidV1 != nil && cidV2 != nil {
m.clearFs([]string{filePath, filePath2})
m.writeLastManifest(cidV1, cidV2, root, height, filePath, filePath2)
cidV2, _, filePath := m.createShapshotForVersion(height, SnapshotVersionV2)
if cidV2 != nil {
m.clearFs([]string{filePath})
m.writeLastManifest(cidV2, root, height, filePath)
}
return root
}

func (m *SnapshotManager) clearFs(excludedFiles []string) {
if prevCid, prevCidV2, _, _, _ := m.repo.LastSnapshotManifest(); prevCid != nil {
m.ipfs.Unpin(prevCid)
if prevCidV2, _, _, _ := m.repo.LastSnapshotManifest(); prevCidV2 != nil {
m.ipfs.Unpin(prevCidV2)
}
m.clearSnapshotFolder(excludedFiles)
Expand Down Expand Up @@ -182,16 +169,12 @@ func (m *SnapshotManager) clearSnapshotFolder(excludedFiles []string) {
}
}

func (m *SnapshotManager) writeLastManifest(snapshotCid []byte, snapshotCidV2 []byte, root common.Hash, height uint64, file string, fileV2 string) {
m.repo.WriteLastSnapshotManifest(snapshotCid, snapshotCidV2, root, height, file, fileV2)
func (m *SnapshotManager) writeLastManifest(snapshotCidV2 []byte, root common.Hash, height uint64, fileV2 string) {
m.repo.WriteLastSnapshotManifest(snapshotCidV2, root, height, fileV2)
}

func (m *SnapshotManager) DownloadSnapshot(snapshot *snapshot.Manifest) (filePath string, version SnapshotVersion, err error) {
version = SnapshotVersionV2
if len(snapshot.CidV2) == 0 {
version = SnapshotVersionV1
}

filePath, file, err := createSnapshotFile(m.cfg.DataDir, snapshot.Height, version)
if err != nil {
return "", 0, err
Expand Down Expand Up @@ -219,13 +202,7 @@ func (m *SnapshotManager) DownloadSnapshot(snapshot *snapshot.Manifest) (filePat
done := make(chan struct{})

go func() {
switch version {
case SnapshotVersionV1:
loadToErr = m.ipfs.LoadTo(snapshot.Cid, file, ctx, onLoading)
case SnapshotVersionV2:
loadToErr = m.ipfs.LoadTo(snapshot.CidV2, file, ctx, onLoading)
}

loadToErr = m.ipfs.LoadTo(snapshot.CidV2, file, ctx, onLoading)
wg.Done()
close(done)
}()
Expand All @@ -252,14 +229,12 @@ func (m *SnapshotManager) DownloadSnapshot(snapshot *snapshot.Manifest) (filePat

if loadToErr == nil {
m.clearFs([]string{filePath})
var filePath1, filePath2 string
if version == SnapshotVersionV1 {
filePath1 = filePath
}
var filePath2 string

if version == SnapshotVersionV2 {
filePath2 = filePath
}
m.writeLastManifest(snapshot.Cid, snapshot.CidV2, snapshot.Root, snapshot.Height, filePath1, filePath2)
m.writeLastManifest(snapshot.CidV2, snapshot.Root, snapshot.Height, filePath2)
}

return filePath, version, loadToErr
Expand Down
2 changes: 0 additions & 2 deletions core/state/snapshot/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ type Manifest struct {

func (m *Manifest) ToBytes() ([]byte, error) {
protoObj := &models.ProtoManifest{
Cid: m.Cid,
Height: m.Height,
Root: m.Root[:],
CidV2: m.CidV2,
Expand All @@ -30,7 +29,6 @@ func (m *Manifest) FromBytes(data []byte) error {
}
m.Root = common.BytesToHash(protoObj.Root)
m.Height = protoObj.Height
m.Cid = protoObj.Cid
m.CidV2 = protoObj.CidV2
return nil
}
10 changes: 0 additions & 10 deletions core/state/statedb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1268,16 +1268,6 @@ func (s *StateDB) RecoverSnapshot2(height uint64, treeRoot common.Hash, from io.
return ReadTreeFrom2(pdb, height, treeRoot, from)
}


func (s *StateDB) WriteSnapshot(height uint64, to io.Writer) (root common.Hash, err error) {
return WriteTreeTo(s.db, height, to)
}

func (s *StateDB) RecoverSnapshot(height uint64, treeRoot common.Hash, from io.Reader) error {
pdb := dbm.NewPrefixDB(s.original, StateDbKeys.BuildDbPrefix(height))
return ReadTreeFrom(pdb, height, treeRoot, from)
}

func (s *StateDB) CommitSnapshot(height uint64, batch dbm.Batch) (dropDb dbm.DB) {
pdb := dbm.NewPrefixDB(s.original, StateDbKeys.BuildDbPrefix(height))

Expand Down
128 changes: 0 additions & 128 deletions core/state/statedb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,134 +248,6 @@ func TestStateGlobal_EmptyBlocksRatio(t *testing.T) {
require.Len(t, stateDb.GetOrNewGlobalObject().data.EmptyBlocksBits.Bytes(), 4)
}

func TestStateDB_WriteSnapshot(t *testing.T) {
db := db.NewMemDB()
stateDb, _ := NewLazy(db)
source := rand.NewSource(1)
rnd := rand.New(source)
for i := 0; i < 300; i++ {
key, _ := crypto.GenerateKeyFromSeed(rnd)
addr := crypto.PubkeyToAddress(key.PublicKey)
stateDb.SetBalance(addr, common.DnaBase)
stateDb.SetState(addr, Human)
}

repo := database.NewRepo(db)
repo.WriteFinalConsensus(common.Hash{0x1})

stateDb.AddInvite(common.Address{}, 1)
stateDb.AddInvite(common.Address{0x1}, 1)

stateDb.Commit(true)

buffer := new(bytes.Buffer)
buffer2 := new(bytes.Buffer)
stateDb.WriteSnapshot(1, buffer)

repo.WriteFinalConsensus(common.Hash{0x2})
repo.WriteCanonicalHash(1, common.Hash{0x2})

repo.WriteBlockHeader(&types.Header{
ProposedHeader: &types.ProposedHeader{
Height: 1,
Flags: types.ValidationFinished,
Upgrade: 1,
IpfsHash: []byte{1, 2, 3, 4},
},
})
time.Sleep(time.Second * 2)
stateDb.WriteSnapshot(1, buffer2)

require.True(t, buffer.Len() > 1000)
require.Equal(t, buffer.Bytes(), buffer2.Bytes())
}

func TestStateDB_RecoverSnapshot(t *testing.T) {
//arrange
database := db.NewMemDB()
stateDb, _ := NewLazy(database)

prevStateDb := stateDb.db

identity := common.Address{}
stateDb.AddInvite(identity, 1)

stateDb.Commit(true)
const AddrsCount = 50000
const Height = uint64(2)

for i := 0; i < AddrsCount; i++ {
addr := common.Address{}
addr.SetBytes(common.ToBytes(uint64(i)))
stateDb.SetNonce(addr, uint32(i+1))
}

stateDb.Commit(true)

var keys [][]byte
var values [][]byte

stateDb.IterateAccounts(func(key []byte, value []byte) bool {
keys = append(keys, key)
values = append(values, value)
return false
})

require.Equal(t, AddrsCount, len(keys))

expectedRoot := stateDb.Root()
stateDb.AddInvite(common.Address{}, 2)

stateDb.Commit(true)
stateDb, _ = NewLazy(database)
stateDb.Load(3)
stateDb.tree.Hash()

//act

buffer := new(bytes.Buffer)
stateDb.WriteSnapshot(Height, buffer)
require.True(t, buffer.Len() > 0)

require.Nil(t, stateDb.RecoverSnapshot(Height, expectedRoot, buffer))

batch := stateDb.original.NewBatch()

dropDb := stateDb.CommitSnapshot(Height, batch)
common.ClearDb(dropDb)
batch.WriteSync()
//assert

require.Equal(t, int64(Height), stateDb.tree.Version())
require.Equal(t, expectedRoot, stateDb.Root())

i := 0
stateDb.IterateAccounts(func(key []byte, value []byte) bool {

require.Equal(t, keys[i], key)
require.Equal(t, values[i], value)
i++
return false
})

require.Equal(t, AddrsCount, i)

cnt := 0

stateDb.IterateIdentities(func(key []byte, value []byte) bool {
addr := common.Address{}
addr.SetBytes(key[1:])
require.Equal(t, addr, identity)
cnt++
return false
})
require.Equal(t, 1, cnt)

it, _ := prevStateDb.Iterator(nil, nil)
defer it.Close()
require.False(t, it.Valid())
}

func TestStateDB_RecoverSnapshot2(t *testing.T) {
//arrange
database := db.NewMemDB()
Expand Down
Loading