diff --git a/packages/bitcore-node/src/models/baseTransaction.ts b/packages/bitcore-node/src/models/baseTransaction.ts index 9844975a4c9..b79453e57d4 100644 --- a/packages/bitcore-node/src/models/baseTransaction.ts +++ b/packages/bitcore-node/src/models/baseTransaction.ts @@ -9,6 +9,7 @@ export interface ITransaction { chain: string; network: string; blockHeight?: number; + _block?: ObjectID; blockHash?: string; blockTime?: Date; blockTimeNormalized?: Date; @@ -32,7 +33,7 @@ export abstract class BaseTransaction extends BaseModel< onConnect() { this.collection.createIndex({ txid: 1 }, { background: true }); this.collection.createIndex({ chain: 1, network: 1, blockHeight: 1 }, { background: true }); - this.collection.createIndex({ blockHash: 1 }, { background: true }); + this.collection.createIndex({ _block: 1 }, { background: true }); this.collection.createIndex({ chain: 1, network: 1, blockTimeNormalized: 1 }, { background: true }); this.collection.createIndex( { wallets: 1, blockTimeNormalized: 1 }, diff --git a/packages/bitcore-node/src/models/block.ts b/packages/bitcore-node/src/models/block.ts index e2498de38bc..98463e1cbe2 100644 --- a/packages/bitcore-node/src/models/block.ts +++ b/packages/bitcore-node/src/models/block.ts @@ -57,7 +57,7 @@ export class BitcoinBlock extends BaseBlock { const previousBlock = await this.collection.findOne({ hash: convertedBlock.previousBlockHash, chain, network }); - await this.collection.bulkWrite([blockOp]); + const bulkInsert = await this.collection.bulkWrite([blockOp]); if (previousBlock) { await this.collection.updateOne( { chain, network, hash: previousBlock.hash }, @@ -68,6 +68,7 @@ export class BitcoinBlock extends BaseBlock { await TransactionStorage.batchImport({ txs: block.transactions, + _block: bulkInsert.upsertedIds[0], blockHash: convertedBlock.hash, blockTime: new Date(time), blockTimeNormalized: new Date(timeNormalized), diff --git a/packages/bitcore-node/src/models/coin.ts b/packages/bitcore-node/src/models/coin.ts index 9718fcb5568..e1b4e572ea1 100644 --- a/packages/bitcore-node/src/models/coin.ts +++ b/packages/bitcore-node/src/models/coin.ts @@ -10,6 +10,7 @@ export interface ICoin { network: string; chain: string; mintTxid: string; + _mintTx: ObjectID; mintIndex: number; mintHeight: number; coinbase: boolean; @@ -18,6 +19,7 @@ export interface ICoin { script: Buffer; wallets: Array; spentTxid: string; + _spentTx?: ObjectID; spentHeight: number; confirmations?: number; sequenceNumber?: number; @@ -35,7 +37,7 @@ export class CoinModel extends BaseModel { ]; onConnect() { - this.collection.createIndex({ mintTxid: 1, mintIndex: 1 }, { background: true }); + this.collection.createIndex({ _mintTx: 1, mintIndex: 1 }, { background: true }); this.collection.createIndex( { address: 1, chain: 1, network: 1 }, { @@ -47,18 +49,18 @@ export class CoinModel extends BaseModel { ); this.collection.createIndex({ address: 1 }, { background: true }); this.collection.createIndex({ chain: 1, network: 1, mintHeight: 1 }, { background: true }); - this.collection.createIndex({ spentTxid: 1 }, { background: true, sparse: true }); + this.collection.createIndex({ _spentTx: 1 }, { background: true, sparse: true }); this.collection.createIndex({ chain: 1, network: 1, spentHeight: 1 }, { background: true }); this.collection.createIndex( { wallets: 1, spentHeight: 1, value: 1, mintHeight: 1 }, { background: true, partialFilterExpression: { 'wallets.0': { $exists: true } } } ); this.collection.createIndex( - { wallets: 1, spentTxid: 1 }, + { wallets: 1, _spentTx: 1 }, { background: true, partialFilterExpression: { 'wallets.0': { $exists: true } } } ); this.collection.createIndex( - { wallets: 1, mintTxid: 1 }, + { wallets: 1, _mintTx: 1 }, { background: true, partialFilterExpression: { 'wallets.0': { $exists: true } } } ); } diff --git a/packages/bitcore-node/src/models/transaction.ts b/packages/bitcore-node/src/models/transaction.ts index 6790e331e74..616ebe7eebc 100644 --- a/packages/bitcore-node/src/models/transaction.ts +++ b/packages/bitcore-node/src/models/transaction.ts @@ -39,7 +39,7 @@ export type TaggedBitcoinTx = BitcoinTransaction & { wallets: Array }; export interface MintOp { updateOne: { filter: { - mintTxid: string; + _mintTx: ObjectID; mintIndex: number; chain: string; network: string; @@ -54,6 +54,7 @@ export interface MintOp { value: number; script: Buffer; spentTxid?: string; + _spentTx?: ObjectID; spentHeight?: SpentHeightIndicators; wallets?: Array; }; @@ -70,13 +71,13 @@ export interface MintOp { export interface SpendOp { updateOne: { filter: { - mintTxid: string; + _mintTx: ObjectID; mintIndex: number; spentHeight: { $lt: SpentHeightIndicators }; chain: string; network: string; }; - update: { $set: { spentTxid: string; spentHeight: number } }; + update: { $set: { _spentTx: ObjectID; spentTxid: string; spentHeight: number } }; }; } @@ -89,6 +90,7 @@ export interface TxOp { network: string; blockHeight: number; blockHash?: string; + _block?: ObjectID; blockTime?: Date; blockTimeNormalized?: Date; coinbase: boolean; @@ -101,7 +103,7 @@ export interface TxOp { wallets: Array; mempoolTime?: Date; }; - $setOnInsert?: TxOp['updateOne']['update']['$set']; + $setOnInsert?: Partial; }; upsert: true; forceServerObjectId: true; @@ -205,6 +207,7 @@ export class TransactionModel extends BaseTransaction { height: number; mempoolTime?: Date; blockTime?: Date; + _block?: ObjectID; blockHash?: string; blockTimeNormalized?: Date; parentChain?: string; @@ -238,7 +241,7 @@ export class TransactionModel extends BaseTransaction { .on('finish', r) ); - this.streamSpendOps({ ...params, spentStream }); + await this.streamSpendOps({ ...params, spentStream }); await new Promise(r => spentStream .pipe(new PruneMempoolStream(chain, network, initialSyncComplete)) @@ -257,10 +260,11 @@ export class TransactionModel extends BaseTransaction { } async streamTxOps(params: { - txs: Array; + txs: Array>; height: number; blockTime?: Date; blockHash?: string; + _block?: ObjectID; blockTimeNormalized?: Date; parentChain?: string; forkHeight?: number; @@ -271,6 +275,7 @@ export class TransactionModel extends BaseTransaction { txStream: Readable; }) { let { + _block, blockHash, blockTime, blockTimeNormalized, @@ -368,6 +373,7 @@ export class TransactionModel extends BaseTransaction { network, blockHeight: height, blockHash, + _block, blockTime, blockTimeNormalized, coinbase: tx.isCoinbase(), @@ -379,6 +385,9 @@ export class TransactionModel extends BaseTransaction { value: tx.outputAmount, wallets, ...(mempoolTime && { mempoolTime }) + }, + $setOnInsert: { + _id: tx._id! } }, upsert: true, @@ -442,8 +451,8 @@ export class TransactionModel extends BaseTransaction { } } - for (let tx of params.txs as Array) { - const coinsForTx = mintBatch.filter(mint => mint.updateOne.filter.mintTxid === tx._hash!); + for (let tx of params.txs as Array>) { + const coinsForTx = mintBatch.filter(mint => mint.updateOne.filter._mintTx === tx._id!); tx.wallets = coinsForTx.reduce((wallets, c) => { wallets = wallets.concat(c.updateOne.update.$set.wallets!); return wallets; @@ -454,7 +463,7 @@ export class TransactionModel extends BaseTransaction { } async streamMintOps(params: { - txs: Array; + txs: Array>; height: number; parentChain?: string; forkHeight?: number; @@ -482,6 +491,7 @@ export class TransactionModel extends BaseTransaction { let mintBatch = new Array(); for (let tx of params.txs) { tx._hash = tx.hash; + tx._id = new ObjectID(); let isCoinbase = tx.isCoinbase(); for (let [index, output] of tx.outputs.entries()) { if ( @@ -505,7 +515,7 @@ export class TransactionModel extends BaseTransaction { mintBatch.push({ updateOne: { filter: { - mintTxid: tx._hash, + _mintTx: tx._id, mintIndex: index, chain, network @@ -544,8 +554,8 @@ export class TransactionModel extends BaseTransaction { mintBatch = new Array(); } - streamSpendOps(params: { - txs: Array; + async streamSpendOps(params: { + txs: Array>; height: number; parentChain?: string; forkHeight?: number; @@ -553,33 +563,43 @@ export class TransactionModel extends BaseTransaction { network: string; spentStream: Readable; }) { - let { chain, network, height, parentChain, forkHeight } = params; + let { chain, network, height, parentChain, forkHeight, txs } = params; if (parentChain && forkHeight && height < forkHeight) { params.spentStream.push(null); return; } let spendOpsBatch = new Array(); - for (let tx of params.txs) { + for (let tx of txs) { if (tx.isCoinbase()) { continue; } for (let input of tx.inputs) { let inputObj = input.toObject(); - const updateQuery = { - updateOne: { - filter: { - mintTxid: inputObj.prevTxId, - mintIndex: inputObj.outputIndex, - spentHeight: { $lt: SpentHeightIndicators.minimum }, - chain, - network - }, - update: { - $set: { spentTxid: tx._hash || tx.hash, spentHeight: height, sequenceNumber: inputObj.sequenceNumber } + const spentTxid = inputObj.prevTxId; + const found = txs.find(t => t._hash === spentTxid); + const spentTx = found || (await TransactionStorage.collection.findOne({ chain, network, txid: spentTxid })); + if (spentTx) { + const updateQuery = { + updateOne: { + filter: { + _mintTx: spentTx!._id!, + mintIndex: inputObj.outputIndex, + spentHeight: { $lt: SpentHeightIndicators.minimum }, + chain, + network + }, + update: { + $set: { + _spentTx: tx._id!, + spentTxid: tx._hash || tx.hash, + spentHeight: height, + sequenceNumber: inputObj.sequenceNumber + } + } } - } - }; - spendOpsBatch.push(updateQuery); + }; + spendOpsBatch.push(updateQuery); + } } if (spendOpsBatch.length > MAX_BATCH_SIZE) { params.spentStream.push(spendOpsBatch); @@ -593,10 +613,11 @@ export class TransactionModel extends BaseTransaction { spendOpsBatch = new Array(); } - async findAllRelatedOutputs(forTx: string) { + async findAllRelatedOutputs(forTx: ObjectID) { const seen = {}; - const getOutputs = (txid: string) => - CoinStorage.collection.find({ mintTxid: txid, mintHeight: { $ne: -3 } }).toArray(); + const getOutputs = async (txid: ObjectID) => { + return CoinStorage.collection.find({ _mintTx: txid, mintHeight: { $ne: -3 } }).toArray(); + }; let batch = await getOutputs(forTx); let allRelatedCoins = new Array(); while (batch.length) { @@ -604,8 +625,8 @@ export class TransactionModel extends BaseTransaction { let newBatch = new Array(); for (const coin of batch) { seen[coin.mintTxid] = true; - if (coin.spentTxid && !seen[coin.spentTxid]) { - const outputs = await getOutputs(coin.spentTxid); + if (coin._spentTx && !seen[coin.spentTxid]) { + const outputs = await getOutputs(coin._spentTx); newBatch = newBatch.concat(outputs); } } @@ -651,34 +672,34 @@ export class TransactionModel extends BaseTransaction { chain, network, spentHeight: SpentHeightIndicators.pending, - mintTxid: { $in: spendOps.map(s => s.updateOne.filter.mintTxid) } + _mintTx: { $in: spendOps.map(s => s.updateOne.filter._mintTx) } }) - .project({ mintTxid: 1, mintIndex: 1, spentTxid: 1 }) + .project({ _mintTx: 1, mintIndex: 1, spentTxid: 1, _spentTx: 1 }) .toArray(); coins = coins.filter( c => spendOps.findIndex( s => - s.updateOne.filter.mintTxid === c.mintTxid && + s.updateOne.filter._mintTx === c._mintTx && s.updateOne.filter.mintIndex === c.mintIndex && s.updateOne.update.$set.spentTxid !== c.spentTxid ) > -1 ); - const invalidatedTxids = Array.from(new Set(coins.map(c => c.spentTxid))); + const invalidatedTxids = Array.from(new Set(coins.map(c => c._spentTx!))); for (const txid of invalidatedTxids) { const allRelatedCoins = await this.findAllRelatedOutputs(txid); - const spentTxids = new Set(allRelatedCoins.filter(c => c.spentTxid).map(c => c.spentTxid)); + const spentTxids = new Set(allRelatedCoins.filter(c => c._spentTx).map(c => c._spentTx!)); const txids = [txid].concat(Array.from(spentTxids)); await Promise.all([ this.collection.update( - { chain, network, txid: { $in: txids } }, + { chain, network, _id: { $in: txids } }, { $set: { blockHeight: SpentHeightIndicators.conflicting } }, { multi: true } ), CoinStorage.collection.update( - { chain, network, mintTxid: { $in: txids } }, + { chain, network, _mintTx: { $in: txids } }, { $set: { mintHeight: SpentHeightIndicators.conflicting } }, { multi: true } ) diff --git a/packages/bitcore-node/test/integration/models/block.spec.ts b/packages/bitcore-node/test/integration/models/block.spec.ts index 055fb1ebc64..ffa51e89556 100644 --- a/packages/bitcore-node/test/integration/models/block.spec.ts +++ b/packages/bitcore-node/test/integration/models/block.spec.ts @@ -333,7 +333,7 @@ describe('Block Model', function() { blockHeight: 6 }); - await TransactionStorage.collection.insertOne({ + const tx1 = await TransactionStorage.collection.insertOne({ txid: 'a2262b524615b6d2f409784ceff898fd46bdde6a584269788c41f26ac4b4919e', chain: 'BTC', network: 'regtest', @@ -350,7 +350,7 @@ describe('Block Model', function() { blockHeight: 7 }); - await TransactionStorage.collection.insertOne({ + const tx2 = await TransactionStorage.collection.insertOne({ txid: '8a351fa9fc3fcd38066b4bf61a8b5f71f08aa224d7a86165557e6da7ee13a826', chain: 'BTC', network: 'regtest', @@ -372,6 +372,7 @@ describe('Block Model', function() { network: 'regtest', chain: 'BTC', mintTxid: 'a2262b524615b6d2f409784ceff898fd46bdde6a584269788c41f26ac4b4919g', + _mintTx: tx1.insertedId, spentTxid: '', mintIndex: 0, spentHeight: SpentHeightIndicators.unspent, @@ -387,6 +388,7 @@ describe('Block Model', function() { network: 'regtest', chain: 'BTC', mintTxid: 'a2262b524615b6d2f409784ceff898fd46bdde6a584269788c41f26ac4b4919e', + _mintTx: tx1.insertedId, spentTxid: '', mintIndex: 0, spentHeight: SpentHeightIndicators.unspent, @@ -401,6 +403,7 @@ describe('Block Model', function() { network: 'regtest', chain: 'BTC', mintTxid: '8a351fa9fc3fcd38066b4bf61a8b5f71f08aa224d7a86165557e6da7ee13a826', + _mintTx: tx2.insertedId, spentTxid: '', mintIndex: 0, spentHeight: SpentHeightIndicators.unspent, @@ -415,6 +418,7 @@ describe('Block Model', function() { network: 'regtest', chain: 'BTC', mintTxid: '8c29860888b915715878b21ce14707a17b43f6c51dfb62a1e736e35bc5d8093f', + _mintTx: tx2.insertedId, mintIndex: 0, spentHeight: 8, mintHeight: 7, diff --git a/packages/bitcore-node/test/integration/models/transaction.spec.ts b/packages/bitcore-node/test/integration/models/transaction.spec.ts index 7d3755538cc..b0ef54529de 100644 --- a/packages/bitcore-node/test/integration/models/transaction.spec.ts +++ b/packages/bitcore-node/test/integration/models/transaction.spec.ts @@ -1,5 +1,7 @@ +import { ObjectId } from 'bson'; import { expect } from 'chai'; import * as crypto from 'crypto'; +import { MongoBound } from '../../../src/models/base'; import { CoinStorage, ICoin } from '../../../src/models/coin'; import { IBtcTransaction, SpendOp, TransactionStorage } from '../../../src/models/transaction'; import { SpentHeightIndicators } from '../../../src/types/Coin'; @@ -59,29 +61,43 @@ describe('Transaction Model', function() { chain, network, blockHeight: 1, + _id: new ObjectId(), txid: '01234' }; + + const blockTx2 = { + chain, + network, + blockHeight: 1, + _id: new ObjectId(), + txid: '123456' + }; + const blockTxOutputs = { chain, network, mintHeight: 1, - mintTxid: '01234', + mintTxid: blockTx.txid, + _mintTx: blockTx._id, mintIndex: 0, spentHeight: -1, - spentTxid: '12345' + spentTxid: '12345', // to be invalidated by blockTx2 + _spentTx: blockTx2._id }; + const block2TxOutputs = { chain, network, mintHeight: 2, - mintTxid: '123456', + mintTxid: blockTx2.txid, + _mintTx: blockTx2._id, mintIndex: 0, spentHeight: -1 }; it('should mark transactions invalid that were in the mempool, but no longer valid', async () => { // insert a valid tx, with a valid output - await TransactionStorage.collection.insertOne(blockTx as IBtcTransaction); + await TransactionStorage.collection.insertOne(blockTx as MongoBound); await CoinStorage.collection.insertOne(blockTxOutputs as ICoin); const chainLength = 1; @@ -94,10 +110,12 @@ describe('Transaction Model', function() { chain, network, mintIndex: blockTxOutputs.mintIndex, - mintTxid: blockTxOutputs.mintTxid, + _mintTx: blockTx._id, spentHeight: { $lt: 0 } }, - update: { $set: { spentHeight: block2TxOutputs.mintHeight, spentTxid: block2TxOutputs.mintTxid } } + update: { + $set: { spentHeight: block2TxOutputs.mintHeight, _spentTx: blockTx2._id, spentTxid: block2TxOutputs.mintTxid } + } } }); @@ -120,12 +138,12 @@ describe('Transaction Model', function() { it('should mark a chain of transactions invalid that were in the mempool, but no longer valid', async () => { // insert a valid tx, with a valid output - await TransactionStorage.collection.insertOne(blockTx as IBtcTransaction); + await TransactionStorage.collection.insertOne(blockTx as MongoBound); await CoinStorage.collection.insertOne(blockTxOutputs as ICoin); const chainLength = 5; const txids = await makeMempoolTxChain(chain, network, blockTxOutputs.spentTxid, chainLength); - const allRelatedCoins = await TransactionStorage.findAllRelatedOutputs(blockTxOutputs.spentTxid); + const allRelatedCoins = await TransactionStorage.findAllRelatedOutputs(blockTx._id); expect(allRelatedCoins.length).to.eq(chainLength); const spentOps = new Array(); @@ -135,10 +153,12 @@ describe('Transaction Model', function() { chain, network, mintIndex: blockTxOutputs.mintIndex, - mintTxid: blockTxOutputs.mintTxid, + _mintTx: blockTx._id, spentHeight: { $lt: 0 } }, - update: { $set: { spentHeight: block2TxOutputs.mintHeight, spentTxid: block2TxOutputs.mintTxid } } + update: { + $set: { spentHeight: block2TxOutputs.mintHeight, spentTxid: block2TxOutputs.mintTxid, _spentTx: blockTx2._id } + } } }); @@ -161,12 +181,12 @@ describe('Transaction Model', function() { it('should mark a massive chain of transactions invalid that were in the mempool, but no longer valid', async () => { // insert a valid tx, with a valid output - await TransactionStorage.collection.insertOne(blockTx as IBtcTransaction); + await TransactionStorage.collection.insertOne(blockTx as MongoBound); await CoinStorage.collection.insertOne(blockTxOutputs as ICoin); const chainLength = 2000; const txids = await makeMempoolTxChain(chain, network, blockTxOutputs.spentTxid, chainLength); - const allRelatedCoins = await TransactionStorage.findAllRelatedOutputs(blockTxOutputs.spentTxid); + const allRelatedCoins = await TransactionStorage.findAllRelatedOutputs(blockTxOutputs._spentTx); expect(allRelatedCoins.length).to.eq(chainLength); const spentOps = new Array(); @@ -176,10 +196,16 @@ describe('Transaction Model', function() { chain, network, mintIndex: blockTxOutputs.mintIndex, - mintTxid: blockTxOutputs.mintTxid, + _mintTx: blockTxOutputs._mintTx, spentHeight: { $lt: 0 } }, - update: { $set: { spentHeight: block2TxOutputs.mintHeight, spentTxid: block2TxOutputs.mintTxid } } + update: { + $set: { + spentHeight: block2TxOutputs.mintHeight, + spentTxid: block2TxOutputs.mintTxid, + _spentTx: blockTxOutputs._mintTx + } + } } }); diff --git a/packages/bitcore-node/test/integration/verification.spec.ts b/packages/bitcore-node/test/integration/verification.spec.ts index e7ac0022cb4..e53e6210529 100644 --- a/packages/bitcore-node/test/integration/verification.spec.ts +++ b/packages/bitcore-node/test/integration/verification.spec.ts @@ -1,4 +1,5 @@ import { expect } from 'chai'; +import { ObjectId } from 'mongodb'; import config from '../../src/config'; import { BitcoinBlockStorage } from '../../src/models/block'; import { CoinStorage } from '../../src/models/coin'; @@ -80,6 +81,7 @@ function addCoin() { network, mintIndex: 0, mintTxid: '0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098', + _mintTx: new ObjectId(), address: '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX', coinbase: true, mintHeight: 1, diff --git a/packages/bitcore-node/test/unit/models/coin.spec.ts b/packages/bitcore-node/test/unit/models/coin.spec.ts index 3d01fe1d0d4..e7d839093d4 100644 --- a/packages/bitcore-node/test/unit/models/coin.spec.ts +++ b/packages/bitcore-node/test/unit/models/coin.spec.ts @@ -19,6 +19,7 @@ describe('Coin Model', function() { network: 'regtest', chain: 'BTC', mintTxid: '81f24ac62a6ffb634b74e6278997f0788f3c64e844453f8831d2a526dc3ecb13', + _mintTx: new ObjectId(), mintIndex: 0, mintHeight: 1, coinbase: true, @@ -56,6 +57,7 @@ describe('Coin Model', function() { network: 'regtest', chain: 'BTC', mintTxid: '81f24ac62a6ffb634b74e6278997f0788f3c64e844453f8831d2a526dc3ecb13', + _mintTx: new ObjectId(), mintIndex: 0, mintHeight: 1, coinbase: true,