Merge branch 'thoroughScan' of github.com:kajoseph/bitcore

Author: kajoseph

packages/bitcore-node/src/models/walletAddress.ts

@@ -37,9 +37,10 @@ export class WalletAddressModel extends BaseModel<IWalletAddress> {
     return JSON.stringify(transform);
   }
 
-  async updateCoins(params: { wallet: IWallet; addresses: string[] }) {
-    const { wallet, addresses } = params;
+  async updateCoins(params: { wallet: IWallet; addresses: string[]; opts?: { reprocess?: boolean } }) {
+    const { wallet, addresses, opts } = params;
     const { chain, network } = wallet;
+    const { reprocess } = opts || {};
 
     class AddressInputStream extends Readable {
       addressBatches: string[][];
@@ -65,19 +66,19 @@ export class WalletAddressModel extends BaseModel<IWalletAddress> {
       }
       async _transform(addressBatch, _, callback) {
         try {
-          let exists = (
-            await WalletAddressStorage.collection
-              .find({ chain, network, wallet: wallet._id, address: { $in: addressBatch } })
-              .project({ address: 1, processed: 1 })
-              .toArray()
-          )
-            .filter(walletAddress => walletAddress.processed)
-            .map(walletAddress => walletAddress.address);
-          this.push(
-            addressBatch.filter(address => {
-              return !exists.includes(address);
-            })
-          );
+          if (reprocess) {
+            this.push(addressBatch);
+          } else {
+            const exists = (
+              await WalletAddressStorage.collection
+                .find({ chain, network, wallet: wallet._id, address: { $in: addressBatch } })
+                .project({ address: 1, processed: 1 })
+                .toArray()
+            )
+              .filter(walletAddress => walletAddress.processed)
+              .map(walletAddress => walletAddress.address);
+            this.push(addressBatch.filter(address => !exists.includes(address)));
+          }
           callback();
         } catch (err) {
           callback(err);
@@ -95,15 +96,13 @@ export class WalletAddressModel extends BaseModel<IWalletAddress> {
         }
         try {
           await WalletAddressStorage.collection.bulkWrite(
-            addressBatch.map(address => {
-              return {
-                insertOne: {
-                  document: { chain, network, wallet: wallet._id, address, processed: false }
-                }
-              };
-            })
-          ),
-            { ordered: false };
+            addressBatch.map(address => ({
+              insertOne: {
+                document: { chain, network, wallet: wallet._id, address, processed: false }
+              }
+            })),
+            { ordered: false }
+          );
         } catch (err: any) {
           // Ignore duplicate keys, they may be half processed
           if (err.code !== 11000) {
@@ -125,14 +124,12 @@ export class WalletAddressModel extends BaseModel<IWalletAddress> {
         }
         try {
           await CoinStorage.collection.bulkWrite(
-            addressBatch.map(address => {
-              return {
-                updateMany: {
-                  filter: { chain, network, address },
-                  update: { $addToSet: { wallets: wallet._id } }
-                }
-              };
-            }),
+            addressBatch.map(address => ({
+              updateMany: {
+                filter: { chain, network, address },
+                update: { $addToSet: { wallets: wallet._id } }
+              }
+            })),
             { ordered: false }
           );
           this.push(addressBatch);

packages/bitcore-node/src/providers/chain-state/internal/internal.ts

@@ -371,8 +371,8 @@ export class InternalStateProvider implements IChainStateService {
   }
 
   async updateWallet(params: UpdateWalletParams) {
-    const { wallet, addresses } = params;
-    await WalletAddressStorage.updateCoins({ wallet, addresses });
+    const { wallet, addresses, reprocess = false } = params;
+    await WalletAddressStorage.updateCoins({ wallet, addresses, opts: { reprocess } });
   }
 
   async streamWalletTransactions(params: StreamWalletTransactionsParams) {

packages/bitcore-node/src/routes/api/wallet.ts

@@ -4,6 +4,7 @@ import logger from '../../logger';
 import { ChainStateProvider } from '../../providers/chain-state';
 import { StreamWalletAddressesParams } from '../../types/namespaces/ChainStateProvider';
 import { Auth, AuthenticatedRequest } from '../../utils/auth';
+import config from '../../config';
 const router = Router({ mergeParams: true });
 
 function isTooLong(field, maxLength = 255) {
@@ -97,15 +98,27 @@ router.get('/:pubKey/check', Auth.authenticateMiddleware, async (req: Authentica
 router.post('/:pubKey', Auth.authenticateMiddleware, async (req: AuthenticatedRequest, res: Response) => {
   let keepAlive;
   try {
-    let { chain, network } = req.params;
-    let addressLines: { address: string }[] = req.body.filter(line => !!line.address);
+    const { chain, network, pubKey } = req.params;
+    const addressLines: { address: string }[] = req.body.filter(line => !!line.address);
 
-    let addresses = addressLines.map(({ address }) => address);
+    const addresses = addressLines.map(({ address }) => address);
     for (const address of addresses) {
       if (isTooLong(address) || !Validation.validateAddress(chain, network, address)) {
         return res.status(413).send('Invalid address');
       }
     }
+    let reprocess = false;
+    if (req.headers['x-reprocess']) {
+      const reprocessOk = Auth.verifyRequestSignature({
+        message: ['reprocess', '/addAddresses' + pubKey, JSON.stringify(req.body)].join('|'),
+        pubKey: config.services.socket.bwsKeys[0],
+        signature: req.headers['x-reprocess']
+      });
+      if (!reprocessOk) {
+        return res.status(401).send('Authentication failed');
+      }
+      reprocess = true;
+    }
     res.status(200);
     keepAlive = setInterval(() => {
       res.write('\n');
@@ -114,7 +127,8 @@ router.post('/:pubKey', Auth.authenticateMiddleware, async (req: AuthenticatedRe
       chain,
       network,
       wallet: req.wallet!,
-      addresses
+      addresses,
+      reprocess
     });
     clearInterval(keepAlive);
     return res.end();

packages/bitcore-node/src/types/namespaces/ChainStateProvider.ts

@@ -84,6 +84,7 @@ export type GetWalletParams = ChainNetwork & PubKey;
 export type UpdateWalletParams = ChainNetwork & {
   wallet: MongoBound<IWallet>;
   addresses: string[];
+  reprocess?: boolean;
 };
 
 export type GetWalletBalanceParams = ChainNetwork & {

packages/bitcore-wallet-client/src/lib/api.ts

@@ -2328,22 +2328,24 @@ export class API extends EventEmitter {
     });
   }
 
-  // /**
-  // * Start an address scanning process.
-  // * When finished, the scanning process will send a notification 'ScanFinished' to all copayers.
-  // *
-  // * @param {Object} opts
-  // * @param {Boolean} opts.includeCopayerBranches (defaults to false)
-  // * @param {Callback} cb
-  // */
+  /**
+   * Start an address scanning process.
+   * When finished, the scanning process will send a notification 'ScanFinished' to all copayers.
+   *
+   * @param {Object} opts
+   * @param {Boolean} opts.includeCopayerBranches (defaults to false)
+   * @param {Number} opts.startIdx (optional) address derivation path start index (support agents only)
+   * @param {Callback} cb
+   */
   startScan(opts, cb) {
     $.checkState(
       this.credentials && this.credentials.isComplete(),
       'Failed state: this.credentials at <startScan()>'
     );
 
-    var args = {
-      includeCopayerBranches: opts.includeCopayerBranches
+    const args = {
+      includeCopayerBranches: opts.includeCopayerBranches,
+      startIdx: opts.startIdx
     };
 
     this.request.post('/v1/addresses/scan', args, err => {
@@ -2460,7 +2462,7 @@ export class API extends EventEmitter {
 
     opts = opts || {};
     var args = [];
-    if (_.isNumber(opts.minTs)) {
+    if (opts.minTs != null && !isNaN(opts.minTs)) {
       args.push('minTs=' + opts.minTs);
     }
     var qs = '';

packages/bitcore-wallet-service/src/lib/blockchainexplorer.ts

@@ -48,7 +48,7 @@ const PROVIDERS = {
   }
 };
 
-export function BlockChainExplorer(opts) {
+export function BlockChainExplorer(opts): V8 {
   $.checkArgument(opts, 'Failed state: opts undefined at <BlockChainExplorer()>');
 
   const provider = opts.provider || 'v8';

packages/bitcore-wallet-service/src/lib/blockchainexplorers/v8.ts

@@ -28,6 +28,8 @@ const Constants = Common.Constants,
   Defaults = Common.Defaults,
   Utils = Common.Utils;
 
+const { sortDesc } = Utils;
+
 function v8network(bwsNetwork, chain = 'btc') {
   if (Utils.getGenericName(bwsNetwork) == 'livenet') return 'mainnet';
   if (Utils.getGenericName(bwsNetwork) == 'testnet' && config.blockchainExplorerOpts?.[chain.toLowerCase()]?.[Utils.getNetworkName(chain.toLowerCase(), 'testnet')]?.regtestEnabled) {
@@ -94,22 +96,23 @@ export class V8 {
     });
   }
 
-  addAddresses(wallet, addresses, cb) {
+  addAddresses(wallet, addresses, cb, opts?) {
     const client = this._getAuthClient(wallet);
+    const payload = addresses.map(a => ({ address: a }));
+    if (opts?.reprocess) {
+      // For peformance, ensure reprocess requests only come from BWS
+      const c = this._getAuthClient({ beAuthPrivateKey2: config.blockchainExplorerOpts.socketApiKey });
+      opts.reprocess = c.sign({ method: 'reprocess', url: 'http://thisdontmatter.com/addAddresses' + wallet.beAuthPublicKey2, payload });
+    }
 
-    const payload = _.map(addresses, a => {
-      return {
-        address: a
-      };
-    });
-
-    const k = 'addAddresses' + addresses.length;
+    const k = 'addAddresses' + !!opts.reprocess + addresses.length;
     const perfKey = getPerformanceKey(k);
     console.time(perfKey);
     client
       .importAddresses({
         payload,
-        pubKey: wallet.beAuthPublicKey2
+        pubKey: wallet.beAuthPublicKey2,
+        reprocess: opts?.reprocess
       })
       .then(ret => {
         console.timeEnd(perfKey);
@@ -278,7 +281,7 @@ export class V8 {
     client
       .getTx({ txid })
       .then(tx => {
-        if (!tx || _.isEmpty(tx)) {
+        if (!tx || JSON.stringify(tx) === '{}') {
           return cb();
         }
         return cb(null, tx);
@@ -327,7 +330,7 @@ export class V8 {
       multisigContractAddress: wallet.multisigContractAddress
     };
 
-    if (_.isNumber(startBlock)) opts.startBlock = startBlock;
+    if (startBlock != null && !isNaN(startBlock)) opts.startBlock = startBlock;
 
     const txStream = client.listTransactions(opts);
     txStream.on('data', raw => {
@@ -341,8 +344,8 @@ export class V8 {
 
       const txs = [],
         unconf = [];
-      _.each(acum.split(/\r?\n/), rawTx => {
-        if (!rawTx) return;
+      for (const rawTx of acum.split(/\r?\n/)) {
+        if (!rawTx) continue;
 
         let tx;
         try {
@@ -356,10 +359,10 @@ export class V8 {
 
         if (tx.height >= 0) txs.push(tx);
         else if (tx.height >= -2) unconf.push(tx);
-      });
+      }
       console.timeEnd(perfKey);
       // blockTime on unconf is 'seenTime';
-      return cb(null, _.flatten(_.orderBy(unconf, 'blockTime', 'desc').concat(txs.reverse())));
+      return cb(null, _.flatten(sortDesc(unconf, 'blockTime').concat(txs.reverse())));
     });
 
     txStream.on('error', e => {

packages/bitcore-wallet-service/src/lib/blockchainexplorers/v8/client.ts

@@ -160,6 +160,9 @@ export class Client {
     logger.debug('addAddresses: %o %o', url, payload);
     const signature = this.sign({ method: 'POST', url, payload });
     const h = { 'x-signature': signature };
+    if (params.reprocess) {
+      h['x-reprocess'] = params.reprocess;
+    }
     return request.post(url, {
       headers: h,
       body: payload,

packages/bitcore-wallet-service/src/lib/chain/btc/index.ts

@@ -38,7 +38,7 @@ export class BtcChain implements IChain {
     return 0;
   }
 
-  getWalletBalance(server, wallet, opts, cb) {
+  getWalletBalance(server: WalletService, wallet, opts, cb) {
     server.getUtxosForCurrentWallet(
       {
         coin: opts.coin,
@@ -74,7 +74,7 @@ export class BtcChain implements IChain {
   }
 
   // opts.payProUrl => only to use different safety margin or not
-  getWalletSendMaxInfo(server, wallet, opts, cb) {
+  getWalletSendMaxInfo(server: WalletService, wallet, opts, cb) {
     server.getUtxosForCurrentWallet({}, (err, utxos) => {
       if (err) return cb(err);
 
@@ -167,7 +167,7 @@ export class BtcChain implements IChain {
     return null;
   }
 
-  getChangeAddress(server, wallet, opts) {
+  getChangeAddress(server: WalletService, wallet, opts) {
     return new Promise((resolve, reject) => {
       const getChangeAddress = (wallet, cb) => {
         if (wallet.singleAddress) {
@@ -339,7 +339,7 @@ export class BtcChain implements IChain {
     return parseInt(fee.toFixed(0));
   }
 
-  getFee(server, wallet, opts) {
+  getFee(server: WalletService, wallet, opts) {
     return new Promise(resolve => {
       server._getFeePerKb(wallet, opts, (err, feePerKb) => {
         return resolve({ feePerKb });
@@ -514,7 +514,7 @@ export class BtcChain implements IChain {
     return bitcoreError;
   }
 
-  checkTxUTXOs(server, txp, opts, cb) {
+  checkTxUTXOs(server: WalletService, txp, opts, cb) {
     logger.debug('Rechecking UTXOs availability for publishTx');
 
     if (txp.replaceTxByFee) {
@@ -561,7 +561,7 @@ export class BtcChain implements IChain {
     return balance;
   }
 
-  selectTxInputs(server, txp, wallet, opts, cb) {
+  selectTxInputs(server: WalletService, txp, wallet, opts, cb) {
     const MAX_TX_SIZE_IN_KB = Defaults.MAX_TX_SIZE_IN_KB_BTC;
 
     // todo: check inputs are ours and have enough value