Merge remote-tracking branch 'origin/staging' into feature/chunks-rpc

Author: ilblackdragon

Cargo.lock

@@ -64,16 +64,16 @@ codegen-units = 1 # Use only 1 codegen-unit to enable full optimizations.
 lto = true
 codegen-units = 1 # Use only 1 codegen-unit to enable full optimizations.
 
-[profile.dev.overrides.hex]
+[profile.dev.package.hex]
 opt-level = 3 # BLS library is too slow to use in debug
 
-[profile.dev.overrides.ring]
+[profile.dev.package.ring]
 opt-level = 3 # BLS library is too slow to use in debug
 
-[profile.dev.overrides.rand]
+[profile.dev.package.rand]
 opt-level = 3 # BLS library is too slow to use in debug
 
-[profile.dev.overrides.bs58]
+[profile.dev.package.bs58]
 opt-level = 3 # bs58 library is too slow to use in debug
 
 [features]

Cargo.toml

@@ -1,5 +1,9 @@
 FROM phusion/baseimage:0.11
 
+RUN apt-get update -qq && apt-get install -y \
+    systemd-coredump \
+    && rm -rf /var/lib/apt/lists/*
+
 EXPOSE 3030 24567
 
 COPY scripts/run_docker.sh /usr/local/bin/run.sh

Dockerfile.prod

@@ -1,5 +1,3 @@
-#![feature(await_macro, async_await)]
-
 use std::time::{Duration, Instant};
 
 use futures03::{compat::Future01CompatExt as _, FutureExt as _};
@@ -10,7 +8,7 @@ use futures03::{compat::Future01CompatExt as _, FutureExt as _};
 /// It is useful when the `futures-preview` is imported as `futures03`.
 macro_rules! select { // replace `::futures_util` with `::futures03` as the crate path
     ($($tokens:tt)*) => {
-        futures03::inner_select::select! {
+        futures03::inner_macro::select! {
             futures_crate_path ( ::futures03 )
             $( $tokens )*
         }

async-utils/src/lib.rs

@@ -1436,7 +1436,7 @@ impl Chain {
 
     /// Returns underlying RuntimeAdapter.
     #[inline]
-    pub fn runtime_adapter(&self) -> Arc<RuntimeAdapter> {
+    pub fn runtime_adapter(&self) -> Arc<dyn RuntimeAdapter> {
         self.runtime_adapter.clone()
     }
 

chain/chain/src/chain.rs

@@ -12,7 +12,7 @@ actix-cors = "0.1"
 base64 = "0.10.0"
 bytes = "0.4.11"
 futures = "0.1"
-futures03 = { package = "futures-preview", version = "0.3.0-alpha.16", features = ["compat", "async-await", "nightly"] }
+futures03 = { package = "futures-preview", version = "0.3.0-alpha.18", features = ["compat", "async-await", "nightly"] }
 chrono = { version = "0.4.4", features = ["serde"] }
 lazy_static = "1.4"
 log = "0.4"

chain/jsonrpc/Cargo.toml

@@ -1,4 +1,3 @@
-#![feature(await_macro, async_await)]
 extern crate prometheus;
 
 use std::convert::TryFrom;

chain/jsonrpc/src/lib.rs

@@ -6,7 +6,7 @@ edition = "2018"
 
 [dependencies]
 sodiumoxide = "0.2.5"
-eth-secp256k1 = { git = "https://github.com/paritytech/rust-secp256k1" }
+parity-secp256k1 = "0.6.0"
 
 bs58 = "0.2.4"
 borsh = "0.2.7"
@@ -15,7 +15,7 @@ serde_derive = "1.0"
 serde_json = "1.0"
 
 lazy_static = "1.4"
-rand = "0.6"
+rand = "0.7"
 libc = "0.2"
 
 [dev-dependencies]

core/crypto/Cargo.toml

@@ -263,7 +263,7 @@ impl SecretKey {
                 SecretKey::ED25519(secret_key)
             }
             KeyType::SECP256K1 => {
-                let mut rng = StdRng::from_rng(OsRng::new().unwrap()).unwrap();
+                let mut rng = StdRng::from_rng(OsRng::default()).unwrap();
                 SecretKey::SECP256K1(secp256k1::key::SecretKey::new(&SECP256K1, &mut rng))
             }
         }
@@ -591,16 +591,16 @@ mod tests {
 
         let sk = SecretKey::from_seed(KeyType::SECP256K1, "test");
         let pk = sk.public_key();
-        let expected = "\"secp256k1:2xVqteU8PWhadHTv99TGh3bSffGb32cFokopSg8xPAiFVjv5UvRUYfNPzmZctBuXryhQosc9dfY1TKKrV4YiyiZH\"";
+        let expected = "\"secp256k1:BtJtBjukUQbcipnS78adSwUKE38sdHnk7pTNZH7miGXfodzUunaAcvY43y37nm7AKbcTQycvdgUzFNWsd7dgPZZ\"";
         assert_eq!(serde_json::to_string(&pk).unwrap(), expected);
         assert_eq!(pk, serde_json::from_str(expected).unwrap());
 
-        let expected = "\"secp256k1:9ZAJB311CURTStj66tvL69fyu6UqWUDNHzdH5Tuixh1Y\"";
+        let expected = "\"secp256k1:9ZNzLxNff6ohoFFGkbfMBAFpZgD7EPoWeiuTpPAeeMRV\"";
         assert_eq!(serde_json::to_string(&sk).unwrap(), expected);
         assert_eq!(sk, serde_json::from_str(expected).unwrap());
 
         let signature = sk.sign(&data);
-        let expected = "\"secp256k1:Ng9nnxbC1BBWJ8NZ8XyExB7kUMtWNn8xCdxRtVC1m7ghH8EpwcupZU3BAxk6ehamaB89aSzhC6UeME4QgZtfkhtwi\"";
+        let expected = "\"secp256k1:7iA75xRmHw17MbUkSpHxBHFVTuJW6jngzbuJPJutwb3EAwVw21wrjpMHU7fFTAqH7D3YEma8utCdvdtsqcAWqnC7r\"";
         assert_eq!(serde_json::to_string(&signature).unwrap(), expected);
         assert_eq!(signature, serde_json::from_str(expected).unwrap());
     }

core/crypto/src/signature.rs

@@ -170,6 +170,7 @@ impl Display for InvalidAccessKeyError {
 #[derive(BorshSerialize, BorshDeserialize, Debug, Clone, PartialEq, Eq)]
 pub struct BalanceMismatchError {
     // Input balances
+    pub incoming_validator_rewards: Balance,
     pub initial_accounts_balance: Balance,
     pub incoming_receipts_balance: Balance,
     pub initial_postponed_receipts_balance: Balance,
@@ -186,7 +187,8 @@ impl Display for BalanceMismatchError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
         // Using saturating add to avoid overflow in display
         let initial_balance = self
-            .initial_accounts_balance
+            .incoming_validator_rewards
+            .saturating_add(self.initial_accounts_balance)
             .saturating_add(self.incoming_receipts_balance)
             .saturating_add(self.initial_postponed_receipts_balance);
         let final_balance = self
@@ -200,6 +202,7 @@ impl Display for BalanceMismatchError {
             f,
             "Balance Mismatch Error. The input balance {} doesn't match output balance {}\n\
              Inputs:\n\
+             \tIncoming validator rewards sum: {}\n\
              \tInitial accounts balance sum: {}\n\
              \tIncoming receipts balance sum: {}\n\
              \tInitial postponed receipts balance sum: {}\n\
@@ -212,6 +215,7 @@ impl Display for BalanceMismatchError {
              \tTotal balance burnt: {}",
             initial_balance,
             final_balance,
+            self.incoming_validator_rewards,
             self.initial_accounts_balance,
             self.incoming_receipts_balance,
             self.initial_postponed_receipts_balance,

core/primitives/src/errors.rs

@@ -12,9 +12,7 @@ use crate::trie::TrieChanges;
 use super::{Trie, TrieIterator};
 use crate::StorageError;
 
-// TODO(#1481): Remove clone after Runtime bug is fixed.
 /// Provides a way to access Storage and record changes with future commit.
-#[derive(Clone)]
 pub struct TrieUpdate {
     pub trie: Arc<Trie>,
     root: MerkleHash,
@@ -276,18 +274,6 @@ mod tests {
 
     use super::*;
 
-    // TODO(#1481): Remove clone after Runtime bug is fixed.
-    #[test]
-    fn trie_clone() {
-        let trie = create_trie();
-        let root = MerkleHash::default();
-        let mut trie_update = TrieUpdate::new(trie.clone(), root);
-        trie_update.set(b"dog".to_vec(), DBValue::from_slice(b"puppy"));
-        let trie_cloned = trie_update.clone();
-        trie_update.set(b"dog".to_vec(), DBValue::from_slice(b"batman"));
-        assert_eq!(trie_cloned.get(b"dog"), Ok(Some(DBValue::from_slice(b"puppy"))));
-    }
-
     #[test]
     fn trie() {
         let trie = create_trie();

core/store/src/trie/update.rs

@@ -1,4 +1,3 @@
-use std::cmp::max;
 use std::collections::{HashMap, HashSet};
 use std::convert::{TryFrom, TryInto};
 use std::fs::File;
@@ -30,12 +29,11 @@ use near_primitives::views::{
     AccessKeyInfoView, CallResult, QueryError, QueryResponse, ViewStateResult,
 };
 use near_store::{
-    get_access_key_raw, get_account, set_account, StorageError, Store, StoreUpdate, Trie,
-    TrieUpdate, WrappedTrieChanges, COL_STATE,
+    get_access_key_raw, Store, StoreUpdate, Trie, TrieUpdate, WrappedTrieChanges, COL_STATE,
 };
 use node_runtime::adapter::ViewRuntimeAdapter;
 use node_runtime::state_viewer::TrieViewer;
-use node_runtime::{ApplyState, Runtime, StateRecord};
+use node_runtime::{ApplyState, Runtime, StateRecord, ValidatorAccountsUpdate};
 
 use crate::config::GenesisConfig;
 use crate::shard_tracker::{account_id_to_shard_id, ShardTracker};
@@ -126,68 +124,6 @@ impl NightshadeRuntime {
         }
     }
 
-    /// Iterates over validator accounts in the given shard and updates their accounts to return stake
-    /// and allocate rewards.
-    fn update_validator_accounts(
-        &self,
-        shard_id: ShardId,
-        block_hash: &CryptoHash,
-        last_validator_proposals: &[ValidatorStake],
-        state_update: &mut TrieUpdate,
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let mut epoch_manager = self.epoch_manager.write().expect(POISONED_LOCK_ERR);
-        let (stake_info, validator_reward) = epoch_manager.compute_stake_return_info(block_hash)?;
-        let account_to_stake =
-            last_validator_proposals.iter().fold(HashMap::new(), |mut acc, v| {
-                acc.insert(v.account_id.clone(), v.amount);
-                acc
-            });
-
-        for (account_id, max_of_stakes) in stake_info {
-            if self.account_id_to_shard_id(&account_id) == shard_id {
-                let account: Option<Account> = get_account(state_update, &account_id)?;
-                if let Some(mut account) = account {
-                    if let Some(reward) = validator_reward.get(&account_id) {
-                        debug!(target: "runtime", "account {} adding reward {} to stake {}", account_id, reward, account.locked);
-                        account.locked += *reward;
-                    }
-
-                    debug!(target: "runtime",
-                           "account {} stake {} max_of_stakes: {}",
-                           account_id, account.locked, max_of_stakes
-                    );
-                    assert!(
-                        account.locked >= max_of_stakes,
-                        "FATAL: staking invariant does not hold. Account stake {} is less than maximum of stakes {} in the past three epochs",
-                        account.locked,
-                        max_of_stakes
-                    );
-                    let last_stake = *account_to_stake.get(&account_id).unwrap_or(&0);
-                    let return_stake = account.locked - max(max_of_stakes, last_stake);
-                    debug!(target: "runtime", "account {} return stake {}", account_id, return_stake);
-                    account.locked -= return_stake;
-                    account.amount += return_stake;
-
-                    set_account(state_update, &account_id, &account);
-                }
-            }
-        }
-        if self.account_id_to_shard_id(&self.genesis_config.protocol_treasury_account) == shard_id {
-            let mut protocol_treasury_account =
-                get_account(state_update, &self.genesis_config.protocol_treasury_account)?.unwrap();
-            protocol_treasury_account.amount +=
-                *validator_reward.get(&self.genesis_config.protocol_treasury_account).unwrap();
-            set_account(
-                state_update,
-                &self.genesis_config.protocol_treasury_account,
-                &protocol_treasury_account,
-            );
-        }
-        state_update.commit();
-
-        Ok(())
-    }
-
     fn genesis_state_from_dump(&self) -> (StoreUpdate, Vec<StateRoot>) {
         let store_update = self.store.store_update();
         let mut state_file = self.home_dir.clone();
@@ -603,33 +539,45 @@ impl RuntimeAdapter for NightshadeRuntime {
         } else {
             self.trie.clone()
         };
-        let mut state_update = TrieUpdate::new(trie.clone(), state_root.hash);
-        let should_update_account = {
+        let validator_accounts_update = {
             let mut epoch_manager = self.epoch_manager.write().expect(POISONED_LOCK_ERR);
             debug!(target: "runtime",
                 "block index: {}, is next_block_epoch_start {}",
                 block_index,
                 epoch_manager.is_next_block_epoch_start(prev_block_hash).unwrap()
             );
-            epoch_manager.is_next_block_epoch_start(prev_block_hash)?
+            if epoch_manager.is_next_block_epoch_start(prev_block_hash)? {
+                let (stake_info, validator_reward) =
+                    epoch_manager.compute_stake_return_info(prev_block_hash)?;
+                let stake_info = stake_info
+                    .into_iter()
+                    .filter(|(account_id, _)| self.account_id_to_shard_id(account_id) == shard_id)
+                    .collect();
+                let validator_rewards = validator_reward
+                    .into_iter()
+                    .filter(|(account_id, _)| self.account_id_to_shard_id(account_id) == shard_id)
+                    .collect();
+                let last_proposals = last_validator_proposals
+                    .iter()
+                    .filter(|v| self.account_id_to_shard_id(&v.account_id) == shard_id)
+                    .fold(HashMap::new(), |mut acc, v| {
+                        acc.insert(v.account_id.clone(), v.amount);
+                        acc
+                    });
+                Some(ValidatorAccountsUpdate {
+                    stake_info,
+                    validator_rewards,
+                    last_proposals,
+                    protocol_treasury_account_id: Some(
+                        self.genesis_config.protocol_treasury_account.clone(),
+                    )
+                    .filter(|account_id| self.account_id_to_shard_id(account_id) == shard_id),
+                })
+            } else {
+                None
+            }
         };
 
-        // If we are starting to apply 1st block in the new epoch.
-        if should_update_account {
-            self.update_validator_accounts(
-                shard_id,
-                prev_block_hash,
-                last_validator_proposals,
-                &mut state_update,
-            )
-            .map_err(|e| {
-                if let Some(e) = e.downcast_ref::<StorageError>() {
-                    panic!(e.to_string())
-                }
-                Error::from(ErrorKind::ValidatorError(e.to_string()))
-            })?;
-        }
-
         let apply_state = ApplyState {
             block_index,
             epoch_length: self.genesis_config.epoch_length,
@@ -639,7 +587,14 @@ impl RuntimeAdapter for NightshadeRuntime {
 
         let apply_result = self
             .runtime
-            .apply(state_update, &apply_state, &receipts, &transactions)
+            .apply(
+                trie.clone(),
+                state_root.hash,
+                &validator_accounts_update,
+                &apply_state,
+                &receipts,
+                &transactions,
+            )
             .map_err(|e| match e {
                 RuntimeError::InvalidTxError(_) => ErrorKind::InvalidTransactions,
                 RuntimeError::BalanceMismatch(e) => panic!("{}", e),

near/src/runtime.rs

@@ -3,7 +3,7 @@ use near::get_store_path;
 use near_primitives::receipt::Receipt;
 use near_primitives::transaction::{ExecutionStatus, SignedTransaction};
 use near_primitives::types::MerkleHash;
-use near_store::{create_store, Trie, TrieUpdate, COL_STATE};
+use near_store::{create_store, Trie, COL_STATE};
 use node_runtime::config::RuntimeConfig;
 use node_runtime::{ApplyState, Runtime};
 use std::fs::File;
@@ -64,10 +64,16 @@ impl RuntimeTestbed {
     }
 
     pub fn process_block(&mut self, transactions: &[SignedTransaction], allow_failures: bool) {
-        let state_update = TrieUpdate::new(self.trie.clone(), self.root);
         let apply_result = self
             .runtime
-            .apply(state_update, &self.apply_state, &self.prev_receipts, transactions)
+            .apply(
+                self.trie.clone(),
+                self.root,
+                &None,
+                &self.apply_state,
+                &self.prev_receipts,
+                transactions,
+            )
             .unwrap();
 
         let (store_update, root) = apply_result.trie_changes.into(self.trie.clone()).unwrap();