From 236a470b9bb85de03338b795577114c0e849200c Mon Sep 17 00:00:00 2001 From: Philemon Ukane Date: Fri, 5 Jan 2024 13:50:35 +0100 Subject: [PATCH 1/2] dcrpg:remove duplicates for votes, misses and treasury table Signed-off-by: Philemon Ukane --- db/dcrpg/indexing.go | 12 +++++++++++ db/dcrpg/internal/stakestmts.go | 29 ++++++++++++++++++++++++++ db/dcrpg/queries.go | 36 +++++++++++++++++++++++++++++++++ db/dcrpg/tables.go | 9 +++++++++ 4 files changed, 86 insertions(+) diff --git a/db/dcrpg/indexing.go b/db/dcrpg/indexing.go index 27b5e3edc..86732261c 100644 --- a/db/dcrpg/indexing.go +++ b/db/dcrpg/indexing.go @@ -394,6 +394,18 @@ func (pgb *ChainDB) deleteDuplicateAgendaVotes() (int64, error) { return deleteDuplicateAgendaVotes(pgb.db) } +func (pgb *ChainDB) DeleteDuplicateVotes() (int64, error) { + return DeleteDuplicateVotes(pgb.db) +} + +func (pgb *ChainDB) DeleteDuplicateMisses() (int64, error) { + return DeleteDuplicateMisses(pgb.db) +} + +func (pgb *ChainDB) DeleteDuplicateTreasuryTxs() (int64, error) { + return DeleteDuplicateTreasuryTxs(pgb.db) +} + // Indexes checks // MissingIndexes lists missing table indexes and their descriptions. diff --git a/db/dcrpg/internal/stakestmts.go b/db/dcrpg/internal/stakestmts.go index 092397684..754974a4b 100644 --- a/db/dcrpg/internal/stakestmts.go +++ b/db/dcrpg/internal/stakestmts.go @@ -414,6 +414,35 @@ const ( FROM agenda_votes) t WHERE t.rnum > 1);` + // DeleteVotesDuplicateRows removes rows that would violate the unique + // index uix_votes_hashes_index. This should be run prior to creating the index. + DeleteVotesDuplicateRows = `DELETE FROM votes + WHERE id IN (SELECT id FROM ( + SELECT id, + row_number() OVER (PARTITION BY tx_hash, block_hash ORDER BY id DESC) AS rnum + FROM votes) t + WHERE t.rnum > 1);` + + // DeleteMissesDuplicateRows removes rows that would violate the unique + // index uix_misses_hashes_index. This should be run prior to creating the index. + DeleteMissesDuplicateRows = `DELETE FROM misses + WHERE id IN (SELECT id FROM ( + SELECT id, + row_number() OVER (PARTITION BY ticket_hash, block_hash ORDER BY id DESC) AS rnum + FROM misses) t + WHERE t.rnum > 1);` + + // DeleteTreasuryTxsDuplicateRows removes rows that would violate the unique + // index uix_treasury_tx_hash. This should be run prior to creating the index. + DeleteTreasuryTxsDuplicateRows = `DELETE FROM treasury + WHERE (tx_hash, block_hash) IN (SELECT tx_hash, block_hash FROM ( + SELECT + tx_hash, + block_hash, + row_number() OVER (PARTITION BY tx_hash, block_hash) AS rnum + FROM treasury) t + WHERE t.rnum > 1);` + // Select SelectAgendasVotesByTime = `SELECT votes.block_time AS timestamp,` + diff --git a/db/dcrpg/queries.go b/db/dcrpg/queries.go index f85afbd0b..2575b97d2 100644 --- a/db/dcrpg/queries.go +++ b/db/dcrpg/queries.go @@ -257,6 +257,42 @@ func deleteDuplicateAgendaVotes(db *sql.DB) (int64, error) { return sqlExec(db, internal.DeleteAgendaVotesDuplicateRows, execErrPrefix) } +// DeleteDuplicateVotes deletes rows in votes with duplicate tx_hash and +// block_hash leaving one row with the lowest id. +func DeleteDuplicateVotes(db *sql.DB) (int64, error) { + if isuniq, err := IsUniqueIndex(db, internal.IndexOfVotesTableOnHashes); err != nil && err != sql.ErrNoRows { + return 0, err + } else if isuniq { + return 0, nil + } + execErrPrefix := "failed to delete duplicate votes: " + return sqlExec(db, internal.DeleteVotesDuplicateRows, execErrPrefix) +} + +// DeleteDuplicateMisses deletes rows in misses with duplicate ticket_hash and +// block_hash leaving one row with the lowest id. +func DeleteDuplicateMisses(db *sql.DB) (int64, error) { + if isuniq, err := IsUniqueIndex(db, internal.IndexOfMissesTableOnHashes); err != nil && err != sql.ErrNoRows { + return 0, err + } else if isuniq { + return 0, nil + } + execErrPrefix := "failed to delete duplicate misses: " + return sqlExec(db, internal.DeleteMissesDuplicateRows, execErrPrefix) +} + +// DeleteDuplicateTreasuryTxs deletes rows in misses with duplicate tx_hash and +// block_hash leaving one row with the lowest id. +func DeleteDuplicateTreasuryTxs(db *sql.DB) (int64, error) { + if isuniq, err := IsUniqueIndex(db, internal.IndexOfTreasuryTableOnTxHash); err != nil && err != sql.ErrNoRows { + return 0, err + } else if isuniq { + return 0, nil + } + execErrPrefix := "failed to delete duplicate treasury txs: " + return sqlExec(db, internal.DeleteTreasuryTxsDuplicateRows, execErrPrefix) +} + // --- stake (votes, tickets, misses, treasury) tables --- func insertTreasuryTxns(db *sql.DB, dbTxns []*dbtypes.Tx, checked, updateExistingRecords bool) error { diff --git a/db/dcrpg/tables.go b/db/dcrpg/tables.go index 72bc72f55..c7bccb4d5 100644 --- a/db/dcrpg/tables.go +++ b/db/dcrpg/tables.go @@ -218,6 +218,15 @@ func (pgb *ChainDB) deleteDuplicates(barLoad chan *dbtypes.ProgressBarLoad) erro // Remove duplicate agenda_votes {TableName: "agenda_votes", DropDupsFunc: pgb.deleteDuplicateAgendaVotes}, + + // Remove duplicate votes + {TableName: "votes", DropDupsFunc: pgb.DeleteDuplicateVotes}, + + // Remove duplicate misses + {TableName: "misses", DropDupsFunc: pgb.DeleteDuplicateMisses}, + + // Remove duplicate treasury txs + {TableName: "treasury", DropDupsFunc: pgb.DeleteDuplicateTreasuryTxs}, } var err error From 3b96c2d0edd882e835f755187bd0ccbd915325d3 Mon Sep 17 00:00:00 2001 From: Philemon Ukane Date: Mon, 27 Oct 2025 10:04:05 +0100 Subject: [PATCH 2/2] unexport rm duplicate func for votes, misses and treasury txs Signed-off-by: Philemon Ukane --- db/dcrpg/indexing.go | 12 ++++++------ db/dcrpg/queries.go | 12 ++++++------ db/dcrpg/tables.go | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/db/dcrpg/indexing.go b/db/dcrpg/indexing.go index 86732261c..b1bffc81b 100644 --- a/db/dcrpg/indexing.go +++ b/db/dcrpg/indexing.go @@ -394,16 +394,16 @@ func (pgb *ChainDB) deleteDuplicateAgendaVotes() (int64, error) { return deleteDuplicateAgendaVotes(pgb.db) } -func (pgb *ChainDB) DeleteDuplicateVotes() (int64, error) { - return DeleteDuplicateVotes(pgb.db) +func (pgb *ChainDB) deleteDuplicateVotes() (int64, error) { + return deleteDuplicateVotes(pgb.db) } -func (pgb *ChainDB) DeleteDuplicateMisses() (int64, error) { - return DeleteDuplicateMisses(pgb.db) +func (pgb *ChainDB) deleteDuplicateMisses() (int64, error) { + return deleteDuplicateMisses(pgb.db) } -func (pgb *ChainDB) DeleteDuplicateTreasuryTxs() (int64, error) { - return DeleteDuplicateTreasuryTxs(pgb.db) +func (pgb *ChainDB) deleteDuplicateTreasuryTxs() (int64, error) { + return deleteDuplicateTreasuryTxs(pgb.db) } // Indexes checks diff --git a/db/dcrpg/queries.go b/db/dcrpg/queries.go index 2575b97d2..cab27fba6 100644 --- a/db/dcrpg/queries.go +++ b/db/dcrpg/queries.go @@ -257,9 +257,9 @@ func deleteDuplicateAgendaVotes(db *sql.DB) (int64, error) { return sqlExec(db, internal.DeleteAgendaVotesDuplicateRows, execErrPrefix) } -// DeleteDuplicateVotes deletes rows in votes with duplicate tx_hash and +// deleteDuplicateVotes deletes rows in votes with duplicate tx_hash and // block_hash leaving one row with the lowest id. -func DeleteDuplicateVotes(db *sql.DB) (int64, error) { +func deleteDuplicateVotes(db *sql.DB) (int64, error) { if isuniq, err := IsUniqueIndex(db, internal.IndexOfVotesTableOnHashes); err != nil && err != sql.ErrNoRows { return 0, err } else if isuniq { @@ -269,9 +269,9 @@ func DeleteDuplicateVotes(db *sql.DB) (int64, error) { return sqlExec(db, internal.DeleteVotesDuplicateRows, execErrPrefix) } -// DeleteDuplicateMisses deletes rows in misses with duplicate ticket_hash and +// deleteDuplicateMisses deletes rows in misses with duplicate ticket_hash and // block_hash leaving one row with the lowest id. -func DeleteDuplicateMisses(db *sql.DB) (int64, error) { +func deleteDuplicateMisses(db *sql.DB) (int64, error) { if isuniq, err := IsUniqueIndex(db, internal.IndexOfMissesTableOnHashes); err != nil && err != sql.ErrNoRows { return 0, err } else if isuniq { @@ -281,9 +281,9 @@ func DeleteDuplicateMisses(db *sql.DB) (int64, error) { return sqlExec(db, internal.DeleteMissesDuplicateRows, execErrPrefix) } -// DeleteDuplicateTreasuryTxs deletes rows in misses with duplicate tx_hash and +// deleteDuplicateTreasuryTxs deletes rows in misses with duplicate tx_hash and // block_hash leaving one row with the lowest id. -func DeleteDuplicateTreasuryTxs(db *sql.DB) (int64, error) { +func deleteDuplicateTreasuryTxs(db *sql.DB) (int64, error) { if isuniq, err := IsUniqueIndex(db, internal.IndexOfTreasuryTableOnTxHash); err != nil && err != sql.ErrNoRows { return 0, err } else if isuniq { diff --git a/db/dcrpg/tables.go b/db/dcrpg/tables.go index c7bccb4d5..433a333b3 100644 --- a/db/dcrpg/tables.go +++ b/db/dcrpg/tables.go @@ -220,13 +220,13 @@ func (pgb *ChainDB) deleteDuplicates(barLoad chan *dbtypes.ProgressBarLoad) erro {TableName: "agenda_votes", DropDupsFunc: pgb.deleteDuplicateAgendaVotes}, // Remove duplicate votes - {TableName: "votes", DropDupsFunc: pgb.DeleteDuplicateVotes}, + {TableName: "votes", DropDupsFunc: pgb.deleteDuplicateVotes}, // Remove duplicate misses - {TableName: "misses", DropDupsFunc: pgb.DeleteDuplicateMisses}, + {TableName: "misses", DropDupsFunc: pgb.deleteDuplicateMisses}, // Remove duplicate treasury txs - {TableName: "treasury", DropDupsFunc: pgb.DeleteDuplicateTreasuryTxs}, + {TableName: "treasury", DropDupsFunc: pgb.deleteDuplicateTreasuryTxs}, } var err error