Introduced chunk API (#1524) (#1537)

frol · ilblackdragon · commit ceff4a490ca1 · 2019-10-23T18:12:09.000-07:00
* Introduced `chunk` API

* Saving genesis chunks to make consistent output from chunks RPC
diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs
@@ -7,6 +7,7 @@ use chrono::prelude::{DateTime, Utc};
 use chrono::Duration;
 use log::{debug, info};
 
+use near_primitives::block::genesis_chunks;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::merkle::{merklize, verify_path};
 use near_primitives::receipt::Receipt;
@@ -185,11 +186,15 @@ impl Chain {
 
         // Get runtime initial state and create genesis block out of it.
         let (state_store_update, state_roots) = runtime_adapter.genesis_state();
-        let genesis = Block::genesis(
+        let genesis_chunks = genesis_chunks(
             state_roots.clone(),
-            chain_genesis.time,
             runtime_adapter.num_shards(),
             chain_genesis.gas_limit,
+        );
+        let genesis = Block::genesis(
+            genesis_chunks.iter().map(|chunk| chunk.header.clone()).collect(),
+            chain_genesis.time,
+            chain_genesis.gas_limit,
             chain_genesis.gas_price,
             chain_genesis.total_supply,
         );
@@ -227,6 +232,9 @@ impl Chain {
             }
             Err(err) => match err.kind() {
                 ErrorKind::DBNotFoundErr(_) => {
+                    for chunk in genesis_chunks {
+                        store_update.save_chunk(&chunk.chunk_hash, chunk.clone());
+                    }
                     runtime_adapter.add_validator_proposals(
                         CryptoHash::default(),
                         genesis.hash(),
@@ -247,7 +255,15 @@ impl Chain {
                         store_update.save_chunk_extra(
                             &genesis.hash(),
                             chunk_header.inner.shard_id,
-                            ChunkExtra::new(state_root, vec![], 0, chain_genesis.gas_limit, 0, 0, 0),
+                            ChunkExtra::new(
+                                state_root,
+                                vec![],
+                                0,
+                                chain_genesis.gas_limit,
+                                0,
+                                0,
+                                0,
+                            ),
                         );
                     }
 
@@ -1652,11 +1668,14 @@ impl<'a> ChainUpdate<'a> {
                         self.chain_store_update.get_chunk_clone_from_header(&chunk_header)?;
 
                     let any_transaction_is_invalid = chunk.transactions.iter().any(|t| {
-                        self.chain_store_update.get_chain_store().check_blocks_on_same_chain(
-                            &block.header,
-                            &t.transaction.block_hash,
-                            self.transaction_validity_period,
-                        ).is_err()
+                        self.chain_store_update
+                            .get_chain_store()
+                            .check_blocks_on_same_chain(
+                                &block.header,
+                                &t.transaction.block_hash,
+                                self.transaction_validity_period,
+                            )
+                            .is_err()
                     });
                     if any_transaction_is_invalid {
                         debug!(target: "chain", "Invalid transactions in the chunk: {:?}", chunk.transactions);
@@ -1693,7 +1712,7 @@ impl<'a> ChainUpdate<'a> {
                             gas_limit,
                             apply_result.total_rent_paid,
                             apply_result.total_validator_reward,
-                            apply_result.total_balance_burnt
+                            apply_result.total_balance_burnt,
                         ),
                     );
                     // Save resulting receipts.
@@ -2200,7 +2219,7 @@ impl<'a> ChainUpdate<'a> {
             gas_limit,
             apply_result.total_rent_paid,
             apply_result.total_validator_reward,
-            apply_result.total_balance_burnt
+            apply_result.total_balance_burnt,
         );
         self.chain_store_update.save_chunk_extra(&block_header.hash, shard_id, chunk_extra);
 
diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs
@@ -483,17 +483,22 @@ mod tests {
     use chrono::Utc;
 
     use near_crypto::{InMemorySigner, KeyType, Signature};
+    use near_primitives::block::genesis_chunks;
 
     use super::*;
 
     #[test]
     fn test_block_produce() {
         let num_shards = 32;
-        let genesis = Block::genesis(
+        let genesis_chunks = genesis_chunks(
             vec![StateRoot { hash: CryptoHash::default(), num_parts: 9 /* TODO MOO */ }],
-            Utc::now(),
             num_shards,
             1_000_000,
+        );
+        let genesis = Block::genesis(
+            genesis_chunks.into_iter().map(|chunk| chunk.header).collect(),
+            Utc::now(),
+            1_000_000,
             100,
             1_000_000_000,
         );
diff --git a/chain/client/src/types.rs b/chain/client/src/types.rs
@@ -255,6 +255,7 @@ impl Message for GetBlock {
 
 /// Actor message requesting a chunk by chunk hash and block hash + shard id.
 pub enum GetChunk {
+    BlockHeight(BlockIndex, ShardId),
     BlockHash(CryptoHash, ShardId),
     ChunkHash(ChunkHash),
 }
diff --git a/chain/client/src/view_client.rs b/chain/client/src/view_client.rs
@@ -174,6 +174,13 @@ impl Handler<GetChunk> for ViewClientActor {
                         .get_chunk(&block.chunks[shard_id as usize].chunk_hash())
                         .map(Clone::clone)
                 })
+            },
+            GetChunk::BlockHeight(block_height, shard_id) => {
+                self.chain.get_block_by_height(block_height).map(Clone::clone).and_then(|block| {
+                    self.chain
+                        .get_chunk(&block.chunks[shard_id as usize].chunk_hash())
+                        .map(Clone::clone)
+                })
             }
         }
         .map(|chunk| chunk.into())
diff --git a/chain/jsonrpc/client/src/lib.rs b/chain/jsonrpc/client/src/lib.rs
@@ -6,9 +6,9 @@ use serde::Deserialize;
 use serde::Serialize;
 
 use near_primitives::hash::CryptoHash;
-use near_primitives::types::BlockIndex;
+use near_primitives::types::{BlockIndex, ShardId};
 use near_primitives::views::{
-    BlockView, ExecutionOutcomeView, FinalExecutionOutcomeView, QueryResponse, StatusResponse,
+    BlockView, ChunkView, ExecutionOutcomeView, FinalExecutionOutcomeView, QueryResponse, StatusResponse,
 };
 
 use crate::message::{from_slice, Message};
@@ -22,6 +22,13 @@ pub enum BlockId {
     Hash(CryptoHash),
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum ChunkId {
+    BlockShardId(BlockId, ShardId),
+    Hash(CryptoHash),
+}
+
 /// Timeout for establishing connection.
 const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
 
@@ -180,6 +187,7 @@ jsonrpc_client!(pub struct JsonRpcClient {
     pub fn tx(&mut self, hash: String) -> RpcRequest<FinalExecutionOutcomeView>;
     pub fn tx_details(&mut self, hash: String) -> RpcRequest<ExecutionOutcomeView>;
     pub fn block(&mut self, id: BlockId) -> RpcRequest<BlockView>;
+    pub fn chunk(&mut self, id: ChunkId) -> RpcRequest<ChunkView>;
 });
 
 /// Create new JSON RPC client that connects to the given address.
diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs
@@ -19,10 +19,10 @@ use async_utils::{delay, timeout};
 use message::Message;
 use message::{Request, RpcError};
 use near_client::{
-    ClientActor, GetBlock, GetNetworkInfo, Query, Status, TxDetails, TxStatus, ViewClientActor,
+    ClientActor, GetBlock, GetChunk, GetNetworkInfo, Query, Status, TxDetails, TxStatus, ViewClientActor,
 };
 pub use near_jsonrpc_client as client;
-use near_jsonrpc_client::{message, BlockId};
+use near_jsonrpc_client::{message, BlockId, ChunkId};
 use near_metrics::{Encoder, TextEncoder};
 use near_network::{NetworkClientMessages, NetworkClientResponses};
 use near_primitives::hash::CryptoHash;
@@ -143,6 +143,7 @@ impl JsonRpcHandler {
             "tx" => self.tx_status(request.params).await,
             "tx_details" => self.tx_details(request.params).await,
             "block" => self.block(request.params).await,
+            "chunk" => self.chunk(request.params).await,
             "network_info" => self.network_info().await,
             _ => Err(RpcError::method_not_found(request.method)),
         }
@@ -239,6 +240,22 @@ impl JsonRpcHandler {
         )
     }
 
+    async fn chunk(&self, params: Option<Value>) -> Result<Value, RpcError> {
+        let (chunk_id,) = parse_params::<(ChunkId,)>(params)?;
+        jsonify(
+            self.view_client_addr
+                .send(match chunk_id {
+                    ChunkId::BlockShardId(block_id, shard_id) => match block_id {
+                        BlockId::Height(block_height) => GetChunk::BlockHeight(block_height, shard_id),
+                        BlockId::Hash(block_hash) => GetChunk::BlockHash(block_hash.into(), shard_id),
+                    },
+                    ChunkId::Hash(chunk_hash) => GetChunk::ChunkHash(chunk_hash.into()),
+                })
+                .compat()
+                .await,
+        )
+    }
+
     async fn network_info(&self) -> Result<Value, RpcError> {
         jsonify(self.client_addr.send(GetNetworkInfo {}).compat().await)
     }
diff --git a/chain/jsonrpc/tests/rpc_query.rs b/chain/jsonrpc/tests/rpc_query.rs
@@ -4,11 +4,13 @@ use actix::System;
 use futures::future;
 use futures::future::Future;
 
+use near_crypto::Signature;
 use near_jsonrpc::client::new_client;
 use near_jsonrpc::test_utils::start_all;
-use near_jsonrpc_client::BlockId;
+use near_jsonrpc_client::{BlockId, ChunkId};
 use near_primitives::hash::CryptoHash;
 use near_primitives::test_utils::init_test_logger;
+use near_primitives::types::ShardId;
 
 /// Retrieve blocks via json rpc
 #[test]
@@ -65,6 +67,55 @@ fn test_block_by_hash() {
     .unwrap();
 }
 
+/// Retrieve blocks via json rpc
+#[test]
+fn test_chunk_by_hash() {
+    init_test_logger();
+
+    System::run(|| {
+        let (_view_client_addr, addr) = start_all(true);
+
+        let mut client = new_client(&format!("http://{}", addr.clone()));
+        actix::spawn(
+            client.chunk(ChunkId::BlockShardId(BlockId::Height(0), ShardId::from(0u64))).then(
+                move |chunk| {
+                    let chunk = chunk.unwrap();
+                    assert_eq!(chunk.header.balance_burnt, 0);
+                    assert_eq!(chunk.header.chunk_hash.as_ref().len(), 32);
+                    assert_eq!(chunk.header.encoded_length, 8);
+                    assert_eq!(chunk.header.encoded_merkle_root.as_ref().len(), 32);
+                    assert_eq!(chunk.header.gas_limit, 1000000);
+                    assert_eq!(chunk.header.gas_used, 0);
+                    assert_eq!(chunk.header.height_created, 0);
+                    assert_eq!(chunk.header.height_included, 0);
+                    assert_eq!(chunk.header.outgoing_receipts_root.as_ref().len(), 32);
+                    assert_eq!(chunk.header.prev_block_hash.as_ref().len(), 32);
+                    assert_eq!(chunk.header.prev_state_num_parts, 17);
+                    assert_eq!(chunk.header.prev_state_root_hash.as_ref().len(), 32);
+                    assert_eq!(chunk.header.rent_paid, 0);
+                    assert_eq!(chunk.header.shard_id, 0);
+                    assert!(if let Signature::ED25519(_) = chunk.header.signature {
+                        true
+                    } else {
+                        false
+                    });
+                    assert_eq!(chunk.header.tx_root.as_ref(), &[0; 32]);
+                    assert_eq!(chunk.header.validator_proposals, vec![]);
+                    assert_eq!(chunk.header.validator_reward, 0);
+                    let mut client = new_client(&format!("http://{}", addr));
+                    client.chunk(ChunkId::Hash(chunk.header.chunk_hash)).then(move |same_chunk| {
+                        let same_chunk = same_chunk.unwrap();
+                        assert_eq!(chunk.header.chunk_hash, same_chunk.header.chunk_hash);
+                        System::current().stop();
+                        future::ok(())
+                    })
+                },
+            ),
+        );
+    })
+    .unwrap();
+}
+
 /// Connect to json rpc and query the client.
 #[test]
 fn test_query() {
diff --git a/core/primitives/benches/serialization.rs b/core/primitives/benches/serialization.rs
@@ -9,7 +9,7 @@ use chrono::Utc;
 
 use near_crypto::{InMemorySigner, KeyType, PublicKey, Signature};
 use near_primitives::account::Account;
-use near_primitives::block::Block;
+use near_primitives::block::{genesis_chunks, Block};
 use near_primitives::hash::CryptoHash;
 use near_primitives::transaction::{Action, SignedTransaction, Transaction, TransferAction};
 use near_primitives::types::{EpochId, StateRoot};
@@ -33,11 +33,15 @@ fn create_transaction() -> SignedTransaction {
 }
 
 fn create_block() -> Block {
-    let genesis = Block::genesis(
+    let genesis_chunks = genesis_chunks(
         vec![StateRoot { hash: CryptoHash::default(), num_parts: 1 /* TODO MOO */ }],
-        Utc::now(),
         1,
         1_000,
+    );
+    let genesis = Block::genesis(
+        genesis_chunks.into_iter().map(|chunk| chunk.header).collect(),
+        Utc::now(),
+        1_000,
         1_000,
         1_000,
     );
diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs
@@ -7,7 +7,7 @@ use near_crypto::{EmptySigner, KeyType, PublicKey, Signature, Signer};
 
 use crate::hash::{hash, CryptoHash};
 use crate::merkle::merklize;
-use crate::sharding::{ChunkHashHeight, ShardChunkHeader};
+use crate::sharding::{ChunkHashHeight, EncodedShardChunk, ShardChunk, ShardChunkHeader};
 use crate::types::{
     Balance, BlockIndex, EpochId, Gas, MerkleHash, ShardId, StateRoot, ValidatorStake,
 };
@@ -242,38 +242,48 @@ pub struct Block {
     pub chunks: Vec<ShardChunkHeader>,
 }
 
+pub fn genesis_chunks(
+    state_roots: Vec<StateRoot>,
+    num_shards: ShardId,
+    initial_gas_limit: Gas,
+) -> Vec<ShardChunk> {
+    assert!(state_roots.len() == 1 || state_roots.len() == (num_shards as usize));
+    (0..num_shards)
+        .map(|i| {
+            let (encoded_chunk, _) = EncodedShardChunk::new(
+                CryptoHash::default(),
+                state_roots[i as usize % state_roots.len()].clone(),
+                0,
+                i,
+                3,
+                1,
+                0,
+                initial_gas_limit,
+                0,
+                0,
+                0,
+                CryptoHash::default(),
+                vec![],
+                &vec![],
+                &vec![],
+                CryptoHash::default(),
+                &EmptySigner {},
+            )
+            .expect("Failed to decode genesis chunk");
+            encoded_chunk.decode_chunk(1).expect("Failed to decode genesis chunk")
+        })
+        .collect()
+}
+
 impl Block {
     /// Returns genesis block for given genesis date and state root.
     pub fn genesis(
-        state_roots: Vec<StateRoot>,
+        chunks: Vec<ShardChunkHeader>,
         timestamp: DateTime<Utc>,
-        num_shards: ShardId,
         initial_gas_limit: Gas,
         initial_gas_price: Balance,
         initial_total_supply: Balance,
     ) -> Self {
-        assert!(state_roots.len() == 1 || state_roots.len() == (num_shards as usize));
-        let chunks = (0..num_shards)
-            .map(|i| {
-                ShardChunkHeader::new(
-                    CryptoHash::default(),
-                    state_roots[i as usize % state_roots.len()].clone(),
-                    CryptoHash::default(),
-                    0,
-                    0,
-                    i,
-                    0,
-                    initial_gas_limit,
-                    0,
-                    0,
-                    0,
-                    CryptoHash::default(),
-                    CryptoHash::default(),
-                    vec![],
-                    &EmptySigner {},
-                )
-            })
-            .collect();
         Block {
             header: BlockHeader::genesis(
                 Block::compute_state_root(&chunks),
diff --git a/core/primitives/src/sharding.rs b/core/primitives/src/sharding.rs
diff --git a/core/primitives/src/views.rs b/core/primitives/src/views.rs
diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs

Original file line number	Diff line number	Diff line change
`@@ -255,6 +255,7 @@ impl Message for GetBlock {`
`255`	`255`
`256`	`256`	`/// Actor message requesting a chunk by chunk hash and block hash + shard id.`
`257`	`257`	`pub enum GetChunk {`
	`258`	`+ BlockHeight(BlockIndex, ShardId),`
`258`	`259`	`BlockHash(CryptoHash, ShardId),`
`259`	`260`	`ChunkHash(ChunkHash),`
`260`	`261`	`}`
Original file line number	Diff line number	Diff line change
`@@ -174,6 +174,13 @@ impl Handler<GetChunk> for ViewClientActor {`
`174`	`174`	`.get_chunk(&block.chunks[shard_id as usize].chunk_hash())`
`175`	`175`	`.map(Clone::clone)`
`176`	`176`	`})`
	`177`	`+ },`
	`178`	`+ GetChunk::BlockHeight(block_height, shard_id) => {`
	`179`	`+ self.chain.get_block_by_height(block_height).map(Clone::clone).and_then(\|block\| {`
	`180`	`+ self.chain`
	`181`	`+ .get_chunk(&block.chunks[shard_id as usize].chunk_hash())`
	`182`	`+ .map(Clone::clone)`
	`183`	`+ })`
`177`	`184`	`}`
`178`	`185`	`}`
`179`	`186`	`.map(\|chunk\| chunk.into())`