diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4b77e6371..55bd7c528 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,35 +4,28 @@ ARG DEBIAN_CODENAME=trixie FROM mcr.microsoft.com/devcontainers/go:${GO_VERSION}-${DEBIAN_CODENAME} ARG PG_MAJOR=16 -ARG MARIADB_MAJOR=12.0.2 USER root -# Add PostgreSQL and MariaDB official repositories (detect codename from /etc/os-release) +# ensure /tmp is writable for apt GPG verification under rootless podman +RUN chmod 1777 /tmp + RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ wget ca-certificates gnupg curl \ && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ && echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo $VERSION_CODENAME)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ - && curl -LsSO https://r.mariadb.com/downloads/mariadb_repo_setup \ - && chmod +x mariadb_repo_setup \ - && ./mariadb_repo_setup --mariadb-server-version="mariadb-${MARIADB_MAJOR}" --skip-maxscale --skip-tools \ - && rm -f mariadb_repo_setup \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ postgresql-${PG_MAJOR} postgresql-client-${PG_MAJOR} \ - mariadb-server mariadb-client \ && rm -rf /var/lib/apt/lists/* COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil -# prepare user-owned data dirs for rootless startup RUN mkdir -p /home/vscode/.local/share/pg/pgdata \ - && mkdir -p /home/vscode/.local/share/mysql \ && mkdir -p /home/vscode/.local/share/tmp \ && chown -R vscode:vscode /home/vscode/.local/share -# Set environment variables based on build args ENV PG_BIN_DIR=/usr/lib/postgresql/${PG_MAJOR}/bin USER vscode diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 984c7fc50..780d3286f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,8 +5,7 @@ "args": { "GO_VERSION": "1.25", "DEBIAN_CODENAME": "trixie", - "PG_MAJOR": "16", - "MARIADB_MAJOR": "12.0.2" + "PG_MAJOR": "16" } }, "customizations": { @@ -27,14 +26,10 @@ "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/singularity,type=bind,consistency=cached,relabel=private", "workspaceFolder": "/workspaces/singularity", "postCreateCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/post-create.sh'", - "postStartCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/start-postgres.sh && ${containerWorkspaceFolder}/.devcontainer/start-mysql.sh'", + "postStartCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/start-postgres.sh'", "remoteUser": "vscode", "containerEnv": { "GOPATH": "/home/vscode/go", - "MYSQL_DATABASE": "singularity", - "MYSQL_USER": "singularity", - "MYSQL_PASSWORD": "singularity", - "MYSQL_SOCKET": "/home/vscode/.local/share/mysql/mysql.sock", "TMPDIR": "/home/vscode/.local/share/tmp", "GOTMPDIR": "/home/vscode/.local/share/tmp", "PGDATA": "/home/vscode/.local/share/pg/pgdata", diff --git a/.devcontainer/init-mysql.sh b/.devcontainer/init-mysql.sh deleted file mode 100755 index b9e3f1afc..000000000 --- a/.devcontainer/init-mysql.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# MariaDB client is required for init - -# Resolve socket path; default to user-owned socket -SOCKET="${MYSQL_SOCKET:-${HOME}/.local/share/mysql/mysql.sock}" - -## Removed one-time init guard; operations below are idempotent - -# Determine root auth flags -MYSQL_ROOT_FLAGS=("-uroot") -if [ -n "${MYSQL_ROOT_PASSWORD:-}" ]; then - MYSQL_ROOT_FLAGS+=("-p${MYSQL_ROOT_PASSWORD}") -fi - -# Wait for server readiness (best effort) -echo "Waiting for MySQL server at socket: $SOCKET" -for i in {1..60}; do - if mariadb-admin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then - echo "MySQL server is ready for init" - break - fi - sleep 1 -done - -# Bail if still unreachable -if ! mariadb-admin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then - echo "MySQL server not reachable, init failed" - exit 1 -fi - -# Create database and user idempotently (MySQL 8+ supports IF NOT EXISTS for users) -DB=${MYSQL_DATABASE:-singularity} -USER=${MYSQL_USER:-singularity} -PASS=${MYSQL_PASSWORD:-singularity} - -echo "Creating database and user: ${USER}@localhost and ${USER}@%" -mariadb --socket="$SOCKET" "${MYSQL_ROOT_FLAGS[@]}" <> /home/vscode/.local/share/pg/pgdata/pg_hba.conf fi -# Initialize MariaDB -if [ ! -d "/home/vscode/.local/share/mysql/data/mysql" ]; then - echo "Initializing MariaDB..." - export TMPDIR=/home/vscode/.local/share/mysql/tmp - mariadb-install-db \ - --datadir=/home/vscode/.local/share/mysql/data \ - --tmpdir=/home/vscode/.local/share/mysql/tmp \ - --auth-root-authentication-method=normal \ - --skip-test-db >/dev/null -fi - -# Start both servers -echo "Starting database servers..." +echo "Starting database server..." .devcontainer/start-postgres.sh -.devcontainer/start-mysql.sh -# Create users (databases will be created during testing as needed) echo "Creating database users..." - -# Postgres setup psql -h localhost -p 55432 -d postgres -c "CREATE USER singularity WITH SUPERUSER CREATEDB CREATEROLE LOGIN;" -# MySQL setup -mariadb --socket=/home/vscode/.local/share/mysql/mysql.sock -uroot </dev/null 2>&1; then - echo "MySQL already running" - exit 0 -fi - -# Start MariaDB server -echo "Starting MySQL server" -mkdir -p "${TMP_DIR}" -touch "${LOG_FILE}" -nohup mariadbd \ - --datadir="${DATA_DIR}" \ - --tmpdir="${TMP_DIR}" \ - --socket="${SOCKET}" \ - --pid-file="${PID_FILE}" \ - --bind-address=127.0.0.1 \ - --port="${PORT}" \ - --skip-name-resolve \ - --log-error="${LOG_FILE}" \ - --innodb-print-all-deadlocks=ON \ - >/dev/null 2>&1 & - -# Wait for MySQL to be ready -for i in {1..60}; do - if [ -S "${SOCKET}" ] && grep -q "ready for connections" "${LOG_FILE}" >/dev/null 2>&1; then - echo "MySQL server is ready" - exit 0 - fi - sleep 1 -done - -echo "MySQL server failed to start" -if [ -f "${LOG_FILE}" ]; then - echo "--- Begin MariaDB error log ---" - tail -n 200 "${LOG_FILE}" || true - echo "--- End MariaDB error log ---" -fi -exit 1 diff --git a/.github/actions/go-test-setup/action.yml b/.github/actions/go-test-setup/action.yml index b7b261fc3..800757b78 100644 --- a/.github/actions/go-test-setup/action.yml +++ b/.github/actions/go-test-setup/action.yml @@ -18,14 +18,6 @@ runs: username: 'singularity' password: 'singularity' database: 'singularity' - - name: Setup MySQL database - uses: shogo82148/actions-setup-mysql@v1 - with: - user: 'singularity' - password: 'singularity' - - name: Create MySQL database - shell: bash - run: mysql -u root -e "create database singularity" - run: | echo "GOTESTFLAGS=$GOTESTFLAGS -timeout=30m" >> $GITHUB_ENV echo "GO386FLAGS=$GO386FLAGS -timeout=30m" >> $GITHUB_ENV diff --git a/cmd/app.go b/cmd/app.go index 38f32a977..a9f7cf455 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -32,10 +32,9 @@ var App = &cli.App{ Name: "singularity", Usage: "A tool for large-scale clients with PB-scale data onboarding to Filecoin network", Description: `Database Backend Support: - Singularity supports multiple database backend: sqlite3, postgres, mysql5.7+ + Singularity supports sqlite3 and postgres as database backends. Use '--database-connection-string' or $DATABASE_CONNECTION_STRING to specify the database connection string. Example for postgres - postgres://user:pass@example.com:5432/dbname - Example for mysql - mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true Example for sqlite3 - sqlite:/absolute/path/to/database.db or - sqlite:relative/path/to/database.db diff --git a/database/connstring.go b/database/connstring.go index f64450e12..0f1188d02 100644 --- a/database/connstring.go +++ b/database/connstring.go @@ -9,7 +9,6 @@ import ( "github.com/cockroachdb/errors" "github.com/glebarez/sqlite" - "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -42,13 +41,7 @@ func open(connString string, config *gorm.Config) (*gorm.DB, io.Closer, error) { } if strings.HasPrefix(connString, "mysql://") { - logger.Info("Opening mysql database") - db, err = gorm.Open(mysql.Open(connString[8:]), config) - if err != nil { - return nil, nil, errors.WithStack(err) - } - closer, err = db.DB() - return db, closer, errors.WithStack(err) + return nil, nil, errors.New("MySQL is no longer supported as of v0.8. Migrate to PostgreSQL -- see docs/installation/upgrade.md") } return nil, nil, ErrDatabaseNotSupported diff --git a/database/connstring_cgo.go b/database/connstring_cgo.go index d56864097..252ae6aef 100644 --- a/database/connstring_cgo.go +++ b/database/connstring_cgo.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/cockroachdb/errors" - "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -42,13 +41,7 @@ func open(connString string, config *gorm.Config) (*gorm.DB, io.Closer, error) { } if strings.HasPrefix(connString, "mysql://") { - logger.Info("Opening mysql database") - db, err = gorm.Open(mysql.Open(connString[8:]), config) - if err != nil { - return nil, nil, errors.WithStack(err) - } - closer, err = db.DB() - return db, closer, errors.WithStack(err) + return nil, nil, errors.New("MySQL is no longer supported as of v0.8. Migrate to PostgreSQL -- see docs/installation/upgrade.md") } return nil, nil, ErrDatabaseNotSupported diff --git a/database/connstring_win32.go b/database/connstring_win32.go index 3c694020a..e25231e88 100644 --- a/database/connstring_win32.go +++ b/database/connstring_win32.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/cockroachdb/errors" - "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -28,13 +27,7 @@ func open(connString string, config *gorm.Config) (*gorm.DB, io.Closer, error) { } if strings.HasPrefix(connString, "mysql://") { - logger.Info("Opening mysql database") - db, err = gorm.Open(mysql.Open(connString[8:]), config) - if err != nil { - return nil, nil, errors.WithStack(err) - } - closer, err = db.DB() - return db, closer, errors.WithStack(err) + return nil, nil, errors.New("MySQL is no longer supported as of v0.8. Migrate to PostgreSQL -- see docs/installation/upgrade.md") } return nil, nil, ErrDatabaseNotSupported diff --git a/database/deadlock_debug.go b/database/deadlock_debug.go deleted file mode 100644 index 1a60a2e17..000000000 --- a/database/deadlock_debug.go +++ /dev/null @@ -1,93 +0,0 @@ -package database - -import ( - "strings" - - "gorm.io/gorm" -) - -// PrintDeadlockInfo prints detailed deadlock information from MySQL/MariaDB InnoDB status. -// This should be called when a deadlock error is detected to help diagnose the issue. -// Returns the deadlock information as a string, or empty string if not available. -func PrintDeadlockInfo(db *gorm.DB) string { - if db.Dialector.Name() != "mysql" { - return "" - } - - // Get InnoDB status - var results []map[string]interface{} - err := db.Raw("SHOW ENGINE INNODB STATUS").Scan(&results).Error - if err != nil || len(results) == 0 { - return "" - } - - // Extract status from result - status, ok := results[0]["Status"].(string) - if !ok { - return "" - } - - // Extract just the deadlock section - if idx := strings.Index(status, "LATEST DETECTED DEADLOCK"); idx >= 0 { - endIdx := strings.Index(status[idx:], "--------\nTRANSACTIONS") - if endIdx > 0 { - return status[idx : idx+endIdx] - } - // If no TRANSACTIONS section found, just return everything after deadlock - return status[idx:] - } - - return "" -} - -// EnableDeadlockLogging enables logging of all deadlocks to the MySQL error log. -// By default, MySQL/MariaDB only logs the most recent deadlock. -// This setting persists until the server is restarted. -func EnableDeadlockLogging(db *gorm.DB) error { - if db.Dialector.Name() != "mysql" { - return nil - } - return db.Exec("SET GLOBAL innodb_print_all_deadlocks = ON").Error -} - -// CheckDeadlockLoggingEnabled checks if innodb_print_all_deadlocks is enabled. -func CheckDeadlockLoggingEnabled(db *gorm.DB) (bool, error) { - if db.Dialector.Name() != "mysql" { - return false, nil - } - var result struct { - VariableName string `gorm:"column:Variable_name"` - Value string `gorm:"column:Value"` - } - err := db.Raw("SHOW VARIABLES LIKE 'innodb_print_all_deadlocks'").Scan(&result).Error - if err != nil { - return false, err - } - return strings.ToLower(result.Value) == "on", nil -} - -// GetDataLockWaits returns current lock wait information from performance_schema. -// This requires MySQL 8.0.30+ or MariaDB 10.5+. -func GetDataLockWaits(db *gorm.DB) ([]map[string]interface{}, error) { - if db.Dialector.Name() != "mysql" { - return nil, nil - } - var results []map[string]interface{} - err := db.Raw("SELECT * FROM performance_schema.data_lock_waits").Scan(&results).Error - return results, err -} - -// GetLockWaitTransactions returns transactions currently waiting for locks. -// This requires MySQL 8.0.30+ or MariaDB 10.5+. -func GetLockWaitTransactions(db *gorm.DB) ([]map[string]interface{}, error) { - if db.Dialector.Name() != "mysql" { - return nil, nil - } - var results []map[string]interface{} - err := db.Raw(` - SELECT * FROM performance_schema.events_transactions_current - WHERE STATE = 'ACTIVE' - AND AUTOCOMMIT = 'NO' - `).Scan(&results).Error - return results, err -} diff --git a/docs/en/cli-reference/README.md b/docs/en/cli-reference/README.md index 3a38b53ca..5f239843f 100644 --- a/docs/en/cli-reference/README.md +++ b/docs/en/cli-reference/README.md @@ -10,10 +10,9 @@ USAGE: DESCRIPTION: Database Backend Support: - Singularity supports multiple database backend: sqlite3, postgres, mysql5.7+ + Singularity supports sqlite3 and postgres as database backends. Use '--database-connection-string' or $DATABASE_CONNECTION_STRING to specify the database connection string. Example for postgres - postgres://user:pass@example.com:5432/dbname - Example for mysql - mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true Example for sqlite3 - sqlite:/absolute/path/to/database.db or - sqlite:relative/path/to/database.db diff --git a/docs/en/installation/deploy-to-production.md b/docs/en/installation/deploy-to-production.md index 187f6a413..cc3449d30 100644 --- a/docs/en/installation/deploy-to-production.md +++ b/docs/en/installation/deploy-to-production.md @@ -8,10 +8,6 @@ Singularity uses `sqlite3` as its default database backend due to its ease of se Connection String Example: `postgres://user:pass@example.com:5432/dbname` -- **MySQL**: - Connection String Example: - `mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true` - ## Using Docker Compose for Deployment If you'd like to quickly deploy Singularity along with a PostgreSQL backend, consider using the provided Docker Compose template: diff --git a/go.mod b/go.mod index ed7045460..dff5eea60 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,6 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/text v0.34.0 - gorm.io/driver/mysql v1.6.0 gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.0 @@ -87,7 +86,6 @@ require ( cloud.google.com/go/auth v0.17.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect @@ -200,7 +198,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.28.0 // indirect - github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gofrs/flock v0.13.0 // indirect diff --git a/go.sum b/go.sum index e1b31d590..e78369dd9 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= @@ -496,8 +494,6 @@ github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0 github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= -github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= -github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -1903,8 +1899,6 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= -gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= diff --git a/handler/dataprep/remove_deadlock_test.go b/handler/dataprep/remove_deadlock_test.go index 87fe1cb56..561784c3c 100644 --- a/handler/dataprep/remove_deadlock_test.go +++ b/handler/dataprep/remove_deadlock_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/model" "github.com/data-preservation-programs/singularity/util/testutil" "github.com/google/uuid" @@ -22,7 +21,6 @@ import ( func TestRemovePreparationNoDeadlock(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { req := require.New(t) - testutil.EnableDeadlockLogging(t, db) // Create test data: multiple preparations sharing one storage const numPreparations = 3 @@ -214,12 +212,6 @@ func TestRemovePreparationNoDeadlock(t *testing.T) { }) if err != nil && updateCtx.Err() == nil { - // If it's a deadlock error, get InnoDB status - if strings.Contains(err.Error(), "Deadlock") { - if deadlockInfo := database.PrintDeadlockInfo(db); deadlockInfo != "" { - t.Logf("\n%s", deadlockInfo) - } - } errChan <- err } }(i) diff --git a/handler/dataprep/remove_test.go b/handler/dataprep/remove_test.go index 0e75319a9..c00efa8ef 100644 --- a/handler/dataprep/remove_test.go +++ b/handler/dataprep/remove_test.go @@ -162,32 +162,3 @@ func TestRemovePreparationHandler_CascadeCycle_Postgres(t *testing.T) { } }) } - -// mysql-only: innodb used to reject the delete with duplicate cascade paths -// test the handler path and expect success, dialect branch is intentional -func TestRemovePreparationHandler_CascadeCycle_MySQL(t *testing.T) { - testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - if db.Dialector.Name() != "mysql" { - t.Skip("Skip non-MySQL dialect") - return - } - prep := model.Preparation{Name: "my-prep"} - require.NoError(t, db.Create(&prep).Error) - tmpMy := t.TempDir() - stor := model.Storage{Name: "my-storage", Type: "local", Path: tmpMy} - require.NoError(t, db.Create(&stor).Error) - sa := model.SourceAttachment{PreparationID: prep.ID, StorageID: stor.ID} - require.NoError(t, db.Create(&sa).Error) - root := model.Directory{AttachmentID: &sa.ID, Name: "", ParentID: nil} - require.NoError(t, db.Create(&root).Error) - d1 := model.Directory{AttachmentID: &sa.ID, Name: "sub", ParentID: &root.ID} - require.NoError(t, db.Create(&d1).Error) - f := model.File{AttachmentID: &sa.ID, DirectoryID: &d1.ID, Path: "sub/a.txt", Size: 1} - require.NoError(t, db.Create(&f).Error) - fr := model.FileRange{FileID: f.ID, Offset: 0, Length: 1} - require.NoError(t, db.Create(&fr).Error) - - err := Default.RemovePreparationHandler(ctx, db, fmt.Sprintf("%d", prep.ID), RemoveRequest{}) - require.NoError(t, err) - }) -} diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go index e30d68d0a..35347236d 100644 --- a/handler/wallet/export_keys.go +++ b/handler/wallet/export_keys.go @@ -176,10 +176,8 @@ func HasPrivateKeyColumn(db *gorm.DB) bool { switch dialect { case "sqlite": db.Raw("SELECT COUNT(*) FROM pragma_table_info('actors') WHERE name = 'private_key'").Scan(&count) - case "postgres": + default: // postgres db.Raw("SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'actors' AND column_name = 'private_key'").Scan(&count) - default: // mysql - db.Raw("SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = 'actors' AND column_name = 'private_key'").Scan(&count) } return count > 0 } diff --git a/model/migrate.go b/model/migrate.go index 0c66fd709..a09ded34e 100644 --- a/model/migrate.go +++ b/model/migrate.go @@ -167,20 +167,12 @@ func migrateFKConstraints(db *gorm.DB) error { var deleteRule string var err error - if dialect == "postgres" { - err = db.Raw(` - SELECT rc.delete_rule - FROM information_schema.referential_constraints rc - JOIN information_schema.table_constraints tc ON rc.constraint_name = tc.constraint_name - WHERE tc.table_name = ? AND tc.constraint_name = ? - `, fk.table, fk.constraint).Scan(&deleteRule).Error - } else if dialect == "mysql" { - err = db.Raw(` - SELECT DELETE_RULE - FROM information_schema.REFERENTIAL_CONSTRAINTS - WHERE TABLE_NAME = ? AND CONSTRAINT_NAME = ? - `, fk.table, fk.constraint).Scan(&deleteRule).Error - } + err = db.Raw(` + SELECT rc.delete_rule + FROM information_schema.referential_constraints rc + JOIN information_schema.table_constraints tc ON rc.constraint_name = tc.constraint_name + WHERE tc.table_name = ? AND tc.constraint_name = ? + `, fk.table, fk.constraint).Scan(&deleteRule).Error if err != nil { // Constraint might not exist yet (new install), skip @@ -200,32 +192,18 @@ func migrateFKConstraints(db *gorm.DB) error { logger.Infow("migrating FK constraint to SET NULL", "table", fk.table, "constraint", fk.constraint) - // Drop and recreate with SET NULL - if dialect == "postgres" { - // Postgres DDL is transactional - wrap DROP+ADD so failure rolls back both - // Use NOT VALID to skip row validation - existing rows were valid under CASCADE, - // so they're still valid under SET NULL. - err = db.Transaction(func(tx *gorm.DB) error { - if err := tx.Exec(`ALTER TABLE ` + fk.table + ` DROP CONSTRAINT ` + fk.constraint).Error; err != nil { - return err - } - return tx.Exec(`ALTER TABLE ` + fk.table + ` ADD CONSTRAINT ` + fk.constraint + - ` FOREIGN KEY (` + fk.column + `) REFERENCES ` + fk.refTable + `(id) ON DELETE SET NULL NOT VALID`).Error - }) - if err != nil { - return errors.Wrapf(err, "failed to migrate constraint %s", fk.constraint) - } - } else if dialect == "mysql" { - // MySQL DDL causes implicit commit, so no transaction benefit here - err = db.Exec(`ALTER TABLE ` + fk.table + ` DROP FOREIGN KEY ` + fk.constraint).Error - if err != nil { - return errors.Wrapf(err, "failed to drop constraint %s", fk.constraint) - } - err = db.Exec(`ALTER TABLE ` + fk.table + ` ADD CONSTRAINT ` + fk.constraint + - ` FOREIGN KEY (` + fk.column + `) REFERENCES ` + fk.refTable + `(id) ON DELETE SET NULL`).Error - if err != nil { - return errors.Wrapf(err, "failed to create constraint %s", fk.constraint) + // Drop and recreate with SET NULL. + // Postgres DDL is transactional -- wrap DROP+ADD so failure rolls back both. + // NOT VALID skips row validation -- existing rows were valid under CASCADE. + err = db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec(`ALTER TABLE ` + fk.table + ` DROP CONSTRAINT ` + fk.constraint).Error; err != nil { + return err } + return tx.Exec(`ALTER TABLE ` + fk.table + ` ADD CONSTRAINT ` + fk.constraint + + ` FOREIGN KEY (` + fk.column + `) REFERENCES ` + fk.refTable + `(id) ON DELETE SET NULL NOT VALID`).Error + }) + if err != nil { + return errors.Wrapf(err, "failed to migrate constraint %s", fk.constraint) } } @@ -403,23 +381,13 @@ func dropDealActorFK(db *gorm.DB) error { constraint := "fk_deals_actor" var exists bool - if dialect == "postgres" { - err := db.Raw(` - SELECT EXISTS ( - SELECT 1 FROM information_schema.table_constraints - WHERE table_name = 'deals' AND constraint_name = ? - )`, constraint).Scan(&exists).Error - if err != nil { - return errors.Wrapf(err, "failed to check constraint %s", constraint) - } - } else if dialect == "mysql" { - err := db.Raw(` - SELECT COUNT(*) > 0 FROM information_schema.TABLE_CONSTRAINTS - WHERE TABLE_NAME = 'deals' AND CONSTRAINT_NAME = ? - `, constraint).Scan(&exists).Error - if err != nil { - return errors.Wrapf(err, "failed to check constraint %s", constraint) - } + err := db.Raw(` + SELECT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE table_name = 'deals' AND constraint_name = ? + )`, constraint).Scan(&exists).Error + if err != nil { + return errors.Wrapf(err, "failed to check constraint %s", constraint) } if !exists { @@ -427,11 +395,7 @@ func dropDealActorFK(db *gorm.DB) error { } logger.Infow("dropping legacy deal-actor FK constraint", "constraint", constraint) - if dialect == "postgres" { - return db.Exec(`ALTER TABLE deals DROP CONSTRAINT ` + constraint).Error - } - // mysql - return db.Exec(`ALTER TABLE deals DROP FOREIGN KEY ` + constraint).Error + return db.Exec(`ALTER TABLE deals DROP CONSTRAINT ` + constraint).Error } // backfillDealWalletID sets wallet_id for existing deals that have a client_id @@ -495,10 +459,6 @@ func stripWalletAssignmentFKs(db *gorm.DB) error { db.Exec(`ALTER TABLE wallet_assignments DROP CONSTRAINT IF EXISTS fk_wallet_assignments_wallet`) db.Exec(`ALTER TABLE wallet_assignments DROP CONSTRAINT IF EXISTS fk_wallet_assignments_preparation`) return nil - case "mysql": - db.Exec(`ALTER TABLE wallet_assignments DROP FOREIGN KEY fk_wallet_assignments_wallet`) - db.Exec(`ALTER TABLE wallet_assignments DROP FOREIGN KEY fk_wallet_assignments_preparation`) - return nil } return nil } diff --git a/service/healthcheck/healthcheck.go b/service/healthcheck/healthcheck.go index cc90079d9..6aeb8fd4d 100644 --- a/service/healthcheck/healthcheck.go +++ b/service/healthcheck/healthcheck.go @@ -195,17 +195,8 @@ func cleanupOrphanedRecords(ctx context.Context, db *gorm.DB) { } } -// execBatchDelete deletes rows where column IS NULL with a batch limit. -// Uses dialect-specific SQL because MariaDB doesn't support LIMIT in IN subqueries. -func execBatchDelete(db *gorm.DB, dialect, table, column string, limit int) *gorm.DB { - switch dialect { - case "mysql": - // MySQL/MariaDB support DELETE ... LIMIT directly - return db.Exec("DELETE FROM "+table+" WHERE "+column+" IS NULL LIMIT ?", limit) - default: - // PostgreSQL and SQLite support LIMIT in subqueries - return db.Exec("DELETE FROM "+table+" WHERE id IN (SELECT id FROM "+table+" WHERE "+column+" IS NULL LIMIT ?)", limit) - } +func execBatchDelete(db *gorm.DB, _, table, column string, limit int) *gorm.DB { + return db.Exec("DELETE FROM "+table+" WHERE id IN (SELECT id FROM "+table+" WHERE "+column+" IS NULL LIMIT ?)", limit) } // Register registers a new worker in the database. It uses the provided context and database connection. diff --git a/service/healthcheck/healthcheck_deadlock_test.go b/service/healthcheck/healthcheck_deadlock_test.go index d6c965ac8..cdaa5555c 100644 --- a/service/healthcheck/healthcheck_deadlock_test.go +++ b/service/healthcheck/healthcheck_deadlock_test.go @@ -2,12 +2,10 @@ package healthcheck import ( "context" - "strings" "sync" "testing" "time" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/model" "github.com/data-preservation-programs/singularity/util/testutil" "github.com/google/uuid" @@ -21,7 +19,6 @@ import ( func TestHealthCheckCleanupNoDeadlock(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { req := require.New(t) - testutil.EnableDeadlockLogging(t, db) // Create test preparation and storage for jobs preparation := model.Preparation{ @@ -129,12 +126,6 @@ func TestHealthCheckCleanupNoDeadlock(t *testing.T) { }) if err != nil && updateCtx.Err() == nil { - // If it's a deadlock error, get InnoDB status - if strings.Contains(err.Error(), "Deadlock") { - if deadlockInfo := database.PrintDeadlockInfo(db); deadlockInfo != "" { - t.Logf("\n%s", deadlockInfo) - } - } errChan <- err } }(i) diff --git a/util/testutil/deadlock.go b/util/testutil/deadlock.go deleted file mode 100644 index f2ea21e45..000000000 --- a/util/testutil/deadlock.go +++ /dev/null @@ -1,36 +0,0 @@ -package testutil - -import ( - "testing" - - "github.com/data-preservation-programs/singularity/database" - "gorm.io/gorm" -) - -// EnableDeadlockLogging enables comprehensive deadlock logging for MySQL/MariaDB tests. -// This should be called early in tests that may encounter deadlocks. -// It will: -// - Enable innodb_print_all_deadlocks (logs all deadlocks to error log, not just the last one) -// - Log the current state of deadlock logging -// -// Note: innodb_print_all_deadlocks requires SUPER privilege and persists until server restart. -func EnableDeadlockLogging(t *testing.T, db *gorm.DB) { - // Try to enable it (may fail if already enabled or insufficient privileges) - err := database.EnableDeadlockLogging(db) - if err != nil { - t.Logf("Note: Could not enable innodb_print_all_deadlocks: %v (may not have SUPER privilege)", err) - } - - // Check if it's enabled - enabled, err := database.CheckDeadlockLoggingEnabled(db) - if err != nil { - t.Logf("Note: Could not check innodb_print_all_deadlocks status: %v", err) - return - } - - if enabled { - t.Logf("Deadlock logging enabled: all deadlocks will be logged to MySQL error log") - } else { - t.Logf("Deadlock logging not enabled: only the most recent deadlock will be available") - } -} diff --git a/util/testutil/testdb.go b/util/testutil/testdb.go index 2cb7b21d1..01e25a347 100644 --- a/util/testutil/testdb.go +++ b/util/testutil/testdb.go @@ -2,4 +2,4 @@ package testutil -var SupportedTestDialects = []string{"sqlite", "mysql", "postgres"} +var SupportedTestDialects = []string{"sqlite", "postgres"} diff --git a/util/testutil/testdb_win32.go b/util/testutil/testdb_win32.go index dbfd133c5..e747bd974 100644 --- a/util/testutil/testdb_win32.go +++ b/util/testutil/testdb_win32.go @@ -2,4 +2,4 @@ package testutil -var SupportedTestDialects = []string{"mysql", "postgres"} +var SupportedTestDialects = []string{"postgres"} diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 8955e69a3..7e274bc57 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -82,81 +82,32 @@ func getTestDB(t *testing.T, dialect string) (db *gorm.DB, closer io.Closer, con require.NoError(t, err) return } - // Use UUID for database names to ensure uniqueness and avoid MySQL's 64-character limit - // Remove hyphens to make it a valid database identifier dbName := "test_" + strings.ReplaceAll(uuid.New().String(), "-", "") - switch dialect { - case "mysql": - socket := os.Getenv("MYSQL_SOCKET") - connStr = "mysql://singularity:singularity@unix(" + socket + ")/mysql?parseTime=true" - case "postgres": - pgPort := os.Getenv("PGPORT") - connStr = "postgres://singularity@localhost:" + pgPort + "/postgres?sslmode=disable" - default: + if dialect != "postgres" { require.Fail(t, "Unsupported dialect: "+dialect) } - // Skip initial connection test - databases will be created during testing - // Create database using shell commands to avoid driver transaction issues - switch dialect { - case "postgres": - // Use createdb command for PostgreSQL with UTF-8 encoding - cmd := exec.Command("createdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", "-E", "UTF8", dbName) - output, err := cmd.CombinedOutput() - if err != nil { - t.Logf("Failed to create PostgreSQL database %s: %v, output: %s", dbName, err, string(output)) - return nil, nil, "" - } - t.Logf("Created PostgreSQL database %s", dbName) - case "mysql": - // Use mysql command for MySQL with UTF-8 character set - socket := os.Getenv("MYSQL_SOCKET") - cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "CREATE DATABASE "+dbName) - output, err := cmd.CombinedOutput() - if err != nil { - t.Logf("Failed to create MySQL database %s: %v, output: %s", dbName, err, string(output)) - return nil, nil, "" - } - t.Logf("Created MySQL database %s", dbName) - default: - t.Logf("Unsupported dialect for shell database creation: %s", dialect) + pgPort := os.Getenv("PGPORT") + connStr = "postgres://singularity@localhost:" + pgPort + "/postgres?sslmode=disable" + cmd := exec.Command("createdb", "-h", "localhost", "-p", pgPort, "-U", "singularity", "-E", "UTF8", dbName) + output, err := cmd.CombinedOutput() + if err != nil { + t.Logf("Failed to create PostgreSQL database %s: %v, output: %s", dbName, err, string(output)) return nil, nil, "" } - // Replace database name in connection string - if strings.Contains(connStr, "postgres?") { - connStr = strings.ReplaceAll(connStr, "postgres?", dbName+"?") - } else if strings.Contains(connStr, "mysql?") { - connStr = strings.ReplaceAll(connStr, "mysql?", dbName+"?") - } + t.Logf("Created PostgreSQL database %s", dbName) + connStr = strings.ReplaceAll(connStr, "postgres?", dbName+"?") var closer2 io.Closer db, closer2, err = database.OpenWithLogger(connStr) if err != nil { t.Logf("Failed to connect to test database %s: %v", dbName, err) - // Cleanup using shell commands - switch dialect { - case "postgres": - cmd := exec.Command("dropdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", dbName) - cmd.Run() // Ignore errors during cleanup - case "mysql": - socket := os.Getenv("MYSQL_SOCKET") - cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "DROP DATABASE "+dbName) - cmd.Run() // Ignore errors during cleanup - } + exec.Command("dropdb", "-h", "localhost", "-p", pgPort, "-U", "singularity", dbName).Run() return nil, nil, "" } closer = CloserFunc(func() error { if closer2 != nil { _ = closer2.Close() } - // Cleanup using shell commands - switch dialect { - case "postgres": - cmd := exec.Command("dropdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", dbName) - cmd.Run() // Ignore errors during cleanup - case "mysql": - socket := os.Getenv("MYSQL_SOCKET") - cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "DROP DATABASE "+dbName) - cmd.Run() // Ignore errors during cleanup - } + exec.Command("dropdb", "-h", "localhost", "-p", pgPort, "-U", "singularity", dbName).Run() return nil }) return