Saving genesis chunks to make consistent output from chunks RPC

ilblackdragon · frol · commit b752c41defd0 · 2019-10-23T02:02:06.000+03:00
diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs
@@ -7,6 +7,7 @@ use chrono::prelude::{DateTime, Utc};
 use chrono::Duration;
 use log::{debug, info};
 
+use near_primitives::block::genesis_chunks;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::merkle::{merklize, verify_path};
 use near_primitives::receipt::Receipt;
@@ -185,11 +186,15 @@ impl Chain {
 
         // Get runtime initial state and create genesis block out of it.
         let (state_store_update, state_roots) = runtime_adapter.genesis_state();
-        let genesis = Block::genesis(
+        let genesis_chunks = genesis_chunks(
             state_roots.clone(),
-            chain_genesis.time,
             runtime_adapter.num_shards(),
             chain_genesis.gas_limit,
+        );
+        let genesis = Block::genesis(
+            genesis_chunks.iter().map(|chunk| chunk.header.clone()).collect(),
+            chain_genesis.time,
+            chain_genesis.gas_limit,
             chain_genesis.gas_price,
             chain_genesis.total_supply,
         );
@@ -227,6 +232,9 @@ impl Chain {
             }
             Err(err) => match err.kind() {
                 ErrorKind::DBNotFoundErr(_) => {
+                    for chunk in genesis_chunks {
+                        store_update.save_chunk(&chunk.chunk_hash, chunk.clone());
+                    }
                     runtime_adapter.add_validator_proposals(
                         CryptoHash::default(),
                         genesis.hash(),
@@ -247,7 +255,15 @@ impl Chain {
                         store_update.save_chunk_extra(
                             &genesis.hash(),
                             chunk_header.inner.shard_id,
-                            ChunkExtra::new(state_root, vec![], 0, chain_genesis.gas_limit, 0, 0, 0),
+                            ChunkExtra::new(
+                                state_root,
+                                vec![],
+                                0,
+                                chain_genesis.gas_limit,
+                                0,
+                                0,
+                                0,
+                            ),
                         );
                     }
 
@@ -1652,11 +1668,14 @@ impl<'a> ChainUpdate<'a> {
                         self.chain_store_update.get_chunk_clone_from_header(&chunk_header)?;
 
                     let any_transaction_is_invalid = chunk.transactions.iter().any(|t| {
-                        self.chain_store_update.get_chain_store().check_blocks_on_same_chain(
-                            &block.header,
-                            &t.transaction.block_hash,
-                            self.transaction_validity_period,
-                        ).is_err()
+                        self.chain_store_update
+                            .get_chain_store()
+                            .check_blocks_on_same_chain(
+                                &block.header,
+                                &t.transaction.block_hash,
+                                self.transaction_validity_period,
+                            )
+                            .is_err()
                     });
                     if any_transaction_is_invalid {
                         debug!(target: "chain", "Invalid transactions in the chunk: {:?}", chunk.transactions);
@@ -1693,7 +1712,7 @@ impl<'a> ChainUpdate<'a> {
                             gas_limit,
                             apply_result.total_rent_paid,
                             apply_result.total_validator_reward,
-                            apply_result.total_balance_burnt
+                            apply_result.total_balance_burnt,
                         ),
                     );
                     // Save resulting receipts.
@@ -2200,7 +2219,7 @@ impl<'a> ChainUpdate<'a> {
             gas_limit,
             apply_result.total_rent_paid,
             apply_result.total_validator_reward,
-            apply_result.total_balance_burnt
+            apply_result.total_balance_burnt,
         );
         self.chain_store_update.save_chunk_extra(&block_header.hash, shard_id, chunk_extra);
 
diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs
@@ -483,17 +483,22 @@ mod tests {
     use chrono::Utc;
 
     use near_crypto::{InMemorySigner, KeyType, Signature};
+    use near_primitives::block::genesis_chunks;
 
     use super::*;
 
     #[test]
     fn test_block_produce() {
         let num_shards = 32;
-        let genesis = Block::genesis(
+        let genesis_chunks = genesis_chunks(
             vec![StateRoot { hash: CryptoHash::default(), num_parts: 9 /* TODO MOO */ }],
-            Utc::now(),
             num_shards,
             1_000_000,
+        );
+        let genesis = Block::genesis(
+            genesis_chunks.into_iter().map(|chunk| chunk.header).collect(),
+            Utc::now(),
+            1_000_000,
             100,
             1_000_000_000,
         );
diff --git a/chain/jsonrpc/tests/rpc_query.rs b/chain/jsonrpc/tests/rpc_query.rs
@@ -73,37 +73,45 @@ fn test_chunk_by_hash() {
     init_test_logger();
 
     System::run(|| {
-        let (_view_client_addr, addr) = start_all(false);
+        let (_view_client_addr, addr) = start_all(true);
 
         let mut client = new_client(&format!("http://{}", addr.clone()));
-        actix::spawn(client.chunk(ChunkId::BlockShardId(BlockId::Height(0), ShardId::from(0u64))).then(move |chunk| {
-            let chunk = chunk.unwrap();
-            assert_eq!(chunk.header.balance_burnt, 0);
-            assert_eq!(chunk.header.chunk_hash.as_ref().len(), 32);
-            assert_eq!(chunk.header.encoded_length, 8);
-            assert_eq!(chunk.header.encoded_merkle_root.as_ref().len(), 32);
-            assert_eq!(chunk.header.gas_limit, 10000000);
-            assert_eq!(chunk.header.gas_used, 0);
-            assert_eq!(chunk.header.height_created, 0);
-            assert_eq!(chunk.header.height_included, 0);
-            assert_eq!(chunk.header.outgoing_receipts_root.as_ref().len(), 32);
-            assert_eq!(chunk.header.prev_block_hash.as_ref().len(), 32);
-            assert_eq!(chunk.header.prev_state_num_parts, 9);
-            assert_eq!(chunk.header.prev_state_root_hash.as_ref().len(), 32);
-            assert_eq!(chunk.header.rent_paid, 0);
-            assert_eq!(chunk.header.shard_id, 0);
-            assert!(if let Signature::ED25519(_) = chunk.header.signature { true } else { false });
-            assert_eq!(chunk.header.tx_root.as_ref(), &[0; 32]);
-            assert_eq!(chunk.header.validator_proposals, vec![]);
-            assert_eq!(chunk.header.validator_reward, 0);
-            let mut client = new_client(&format!("http://{}", addr));
-            client.chunk(ChunkId::Hash(chunk.header.chunk_hash)).then(move |same_chunk| {
-                let same_chunk = same_chunk.unwrap();
-                assert_eq!(chunk.header.chunk_hash, same_chunk.header.chunk_hash);
-                System::current().stop();
-                future::ok(())
-            })
-        }));
+        actix::spawn(
+            client.chunk(ChunkId::BlockShardId(BlockId::Height(0), ShardId::from(0u64))).then(
+                move |chunk| {
+                    let chunk = chunk.unwrap();
+                    assert_eq!(chunk.header.balance_burnt, 0);
+                    assert_eq!(chunk.header.chunk_hash.as_ref().len(), 32);
+                    assert_eq!(chunk.header.encoded_length, 8);
+                    assert_eq!(chunk.header.encoded_merkle_root.as_ref().len(), 32);
+                    assert_eq!(chunk.header.gas_limit, 1000000);
+                    assert_eq!(chunk.header.gas_used, 0);
+                    assert_eq!(chunk.header.height_created, 0);
+                    assert_eq!(chunk.header.height_included, 0);
+                    assert_eq!(chunk.header.outgoing_receipts_root.as_ref().len(), 32);
+                    assert_eq!(chunk.header.prev_block_hash.as_ref().len(), 32);
+                    assert_eq!(chunk.header.prev_state_num_parts, 17);
+                    assert_eq!(chunk.header.prev_state_root_hash.as_ref().len(), 32);
+                    assert_eq!(chunk.header.rent_paid, 0);
+                    assert_eq!(chunk.header.shard_id, 0);
+                    assert!(if let Signature::ED25519(_) = chunk.header.signature {
+                        true
+                    } else {
+                        false
+                    });
+                    assert_eq!(chunk.header.tx_root.as_ref(), &[0; 32]);
+                    assert_eq!(chunk.header.validator_proposals, vec![]);
+                    assert_eq!(chunk.header.validator_reward, 0);
+                    let mut client = new_client(&format!("http://{}", addr));
+                    client.chunk(ChunkId::Hash(chunk.header.chunk_hash)).then(move |same_chunk| {
+                        let same_chunk = same_chunk.unwrap();
+                        assert_eq!(chunk.header.chunk_hash, same_chunk.header.chunk_hash);
+                        System::current().stop();
+                        future::ok(())
+                    })
+                },
+            ),
+        );
     })
     .unwrap();
 }
diff --git a/core/primitives/benches/serialization.rs b/core/primitives/benches/serialization.rs
@@ -9,7 +9,7 @@ use chrono::Utc;
 
 use near_crypto::{InMemorySigner, KeyType, PublicKey, Signature};
 use near_primitives::account::Account;
-use near_primitives::block::Block;
+use near_primitives::block::{genesis_chunks, Block};
 use near_primitives::hash::CryptoHash;
 use near_primitives::transaction::{Action, SignedTransaction, Transaction, TransferAction};
 use near_primitives::types::{EpochId, StateRoot};
@@ -33,11 +33,15 @@ fn create_transaction() -> SignedTransaction {
 }
 
 fn create_block() -> Block {
-    let genesis = Block::genesis(
+    let genesis_chunks = genesis_chunks(
         vec![StateRoot { hash: CryptoHash::default(), num_parts: 1 /* TODO MOO */ }],
-        Utc::now(),
         1,
         1_000,
+    );
+    let genesis = Block::genesis(
+        genesis_chunks.into_iter().map(|chunk| chunk.header).collect(),
+        Utc::now(),
+        1_000,
         1_000,
         1_000,
     );
diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs
@@ -7,7 +7,7 @@ use near_crypto::{EmptySigner, KeyType, PublicKey, Signature, Signer};
 
 use crate::hash::{hash, CryptoHash};
 use crate::merkle::merklize;
-use crate::sharding::{ChunkHashHeight, ShardChunkHeader};
+use crate::sharding::{ChunkHashHeight, EncodedShardChunk, ShardChunk, ShardChunkHeader};
 use crate::types::{
     Balance, BlockIndex, EpochId, Gas, MerkleHash, ShardId, StateRoot, ValidatorStake,
 };
@@ -242,38 +242,48 @@ pub struct Block {
     pub chunks: Vec<ShardChunkHeader>,
 }
 
+pub fn genesis_chunks(
+    state_roots: Vec<StateRoot>,
+    num_shards: ShardId,
+    initial_gas_limit: Gas,
+) -> Vec<ShardChunk> {
+    assert!(state_roots.len() == 1 || state_roots.len() == (num_shards as usize));
+    (0..num_shards)
+        .map(|i| {
+            let (encoded_chunk, _) = EncodedShardChunk::new(
+                CryptoHash::default(),
+                state_roots[i as usize % state_roots.len()].clone(),
+                0,
+                i,
+                3,
+                1,
+                0,
+                initial_gas_limit,
+                0,
+                0,
+                0,
+                CryptoHash::default(),
+                vec![],
+                &vec![],
+                &vec![],
+                CryptoHash::default(),
+                &EmptySigner {},
+            )
+            .expect("Failed to decode genesis chunk");
+            encoded_chunk.decode_chunk(1).expect("Failed to decode genesis chunk")
+        })
+        .collect()
+}
+
 impl Block {
     /// Returns genesis block for given genesis date and state root.
     pub fn genesis(
-        state_roots: Vec<StateRoot>,
+        chunks: Vec<ShardChunkHeader>,
         timestamp: DateTime<Utc>,
-        num_shards: ShardId,
         initial_gas_limit: Gas,
         initial_gas_price: Balance,
         initial_total_supply: Balance,
     ) -> Self {
-        assert!(state_roots.len() == 1 || state_roots.len() == (num_shards as usize));
-        let chunks = (0..num_shards)
-            .map(|i| {
-                ShardChunkHeader::new(
-                    CryptoHash::default(),
-                    state_roots[i as usize % state_roots.len()].clone(),
-                    CryptoHash::default(),
-                    0,
-                    0,
-                    i,
-                    0,
-                    initial_gas_limit,
-                    0,
-                    0,
-                    0,
-                    CryptoHash::default(),
-                    CryptoHash::default(),
-                    vec![],
-                    &EmptySigner {},
-                )
-            })
-            .collect();
         Block {
             header: BlockHeader::genesis(
                 Block::compute_state_root(&chunks),
diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs
@@ -1,11 +1,20 @@
 //! Tools for creating a genesis block.
 
+use std::collections::BTreeMap;
+use std::fs::File;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
 use borsh::BorshSerialize;
 use indicatif::{ProgressBar, ProgressStyle};
+use tempdir::TempDir;
+
 use near::{get_store_path, GenesisConfig, NightshadeRuntime};
 use near_chain::{Block, ChainStore, RuntimeAdapter, Tip};
 use near_crypto::{InMemorySigner, KeyType};
 use near_primitives::account::AccessKey;
+use near_primitives::block::genesis_chunks;
 use near_primitives::contract::ContractCode;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::serialize::to_base64;
@@ -15,12 +24,6 @@ use near_store::{
     create_store, get_account, set_access_key, set_account, set_code, Store, TrieUpdate, COL_STATE,
 };
 use node_runtime::StateRecord;
-use std::collections::BTreeMap;
-use std::fs::File;
-use std::io::Write;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use tempdir::TempDir;
 
 fn get_account_id(account_index: u64) -> String {
     format!("near_{}_{}", account_index, account_index)
@@ -179,11 +182,15 @@ impl GenesisBuilder {
     }
 
     fn write_genesis_block(&mut self) -> Result<()> {
-        let genesis = Block::genesis(
+        let genesis_chunks = genesis_chunks(
             self.roots.values().cloned().collect(),
-            self.config.genesis_time,
             self.runtime.num_shards(),
             self.config.gas_limit,
+        );
+        let genesis = Block::genesis(
+            genesis_chunks.into_iter().map(|chunk| chunk.header).collect(),
+            self.config.genesis_time,
+            self.config.gas_limit,
             self.config.gas_price,
             self.config.total_supply,
         );
