From 9aaacc2577cefd604debe0ded52713c26aebb1c5 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Wed, 1 Apr 2026 03:54:10 +0200 Subject: [PATCH 01/25] migrate from MySQL to PostgreSQL Co-authored-by: Copilot --- DiscordBot/DiscordBot.csproj | 2 +- DiscordBot/Extensions/CasinoRepository.cs | 8 +- .../Extensions/DBConnectionExtension.cs | 3 +- DiscordBot/Extensions/UserDBRepository.cs | 28 ++-- DiscordBot/Modules/ModerationModule.cs | 15 +- DiscordBot/Program.cs | 2 + DiscordBot/Services/DatabaseService.cs | 100 +++++-------- DiscordBot/Services/KarmaResetService.cs | 136 ++++++++++++++++++ DiscordBot/Settings/Settings.example.json | 3 +- README.md | 60 ++++++-- docker-compose.yml | 21 ++- .../done/mysql-to-postgresql-migration.md | 49 +++++++ k8s/dev/bot-config.yaml | 2 +- k8s/dev/bot.yaml | 8 +- k8s/dev/external-secrets.yaml | 43 ++---- k8s/dev/mysql-backup.yaml | 63 ++++---- k8s/dev/mysql.yaml | 61 ++++---- k8s/dev/pgadmin.yaml | 99 +++++++++++++ k8s/dev/phpmyadmin.yaml | 45 +++--- k8s/dev/postgresql-backup.yaml | 66 +++++++++ k8s/dev/postgresql.yaml | 110 ++++++++++++++ k8s/prod/bot-config.yaml | 3 +- k8s/prod/bot.yaml | 8 +- k8s/prod/external-secrets.yaml | 43 ++---- k8s/prod/mysql-backup.yaml | 63 ++++---- k8s/prod/mysql.yaml | 61 ++++---- k8s/prod/pgadmin.yaml | 99 +++++++++++++ k8s/prod/phpmyadmin.yaml | 45 +++--- k8s/prod/postgresql-backup.yaml | 66 +++++++++ k8s/prod/postgresql.yaml | 110 ++++++++++++++ 30 files changed, 1051 insertions(+), 371 deletions(-) create mode 100644 DiscordBot/Services/KarmaResetService.cs create mode 100644 docs/plans/done/mysql-to-postgresql-migration.md create mode 100644 k8s/dev/pgadmin.yaml create mode 100644 k8s/dev/postgresql-backup.yaml create mode 100644 k8s/dev/postgresql.yaml create mode 100644 k8s/prod/pgadmin.yaml create mode 100644 k8s/prod/postgresql-backup.yaml create mode 100644 k8s/prod/postgresql.yaml diff --git a/DiscordBot/DiscordBot.csproj b/DiscordBot/DiscordBot.csproj index dc5c4509..97731195 100644 --- a/DiscordBot/DiscordBot.csproj +++ b/DiscordBot/DiscordBot.csproj @@ -7,11 +7,11 @@ + - diff --git a/DiscordBot/Extensions/CasinoRepository.cs b/DiscordBot/Extensions/CasinoRepository.cs index e63c145c..8e8c1c09 100644 --- a/DiscordBot/Extensions/CasinoRepository.cs +++ b/DiscordBot/Extensions/CasinoRepository.cs @@ -8,8 +8,8 @@ public interface ICasinoRepo // Casino User Operations [Sql($@" INSERT INTO {CasinoProps.CasinoTableName} ({CasinoProps.UserID}, {CasinoProps.Tokens}, {CasinoProps.CreatedAt}, {CasinoProps.UpdatedAt}, {CasinoProps.LastDailyReward}) - VALUES (@{CasinoProps.UserID}, @{CasinoProps.Tokens}, @{CasinoProps.CreatedAt}, @{CasinoProps.UpdatedAt}, @{CasinoProps.LastDailyReward}); - SELECT * FROM {CasinoProps.CasinoTableName} WHERE {CasinoProps.UserID} = @{CasinoProps.UserID}")] + VALUES (@{CasinoProps.UserID}, @{CasinoProps.Tokens}, @{CasinoProps.CreatedAt}, @{CasinoProps.UpdatedAt}, @{CasinoProps.LastDailyReward}) + RETURNING *")] Task InsertCasinoUser(CasinoUser user); [Sql($"SELECT * FROM {CasinoProps.CasinoTableName} WHERE {CasinoProps.UserID} = @userId")] @@ -33,8 +33,8 @@ public interface ICasinoRepo // Token Transaction Operations [Sql($@" INSERT INTO {CasinoProps.TransactionTableName} ({CasinoProps.TransactionUserID}, {CasinoProps.Amount}, {CasinoProps.TransactionType}, {CasinoProps.Details}, {CasinoProps.TransactionCreatedAt}) - VALUES (@{CasinoProps.TransactionUserID}, @{CasinoProps.Amount}, @{CasinoProps.TransactionType}, @{CasinoProps.Details}, @{CasinoProps.TransactionCreatedAt}); - SELECT * FROM {CasinoProps.TransactionTableName} WHERE {CasinoProps.TransactionId} = LAST_INSERT_ID()")] + VALUES (@{CasinoProps.TransactionUserID}, @{CasinoProps.Amount}, @{CasinoProps.TransactionType}, @{CasinoProps.Details}::jsonb, @{CasinoProps.TransactionCreatedAt}) + RETURNING *")] Task InsertTransaction(TokenTransaction tokenTransaction); [Sql($"SELECT * FROM {CasinoProps.TransactionTableName} WHERE {CasinoProps.TransactionUserID} = @userId ORDER BY {CasinoProps.TransactionCreatedAt} DESC LIMIT @limit")] diff --git a/DiscordBot/Extensions/DBConnectionExtension.cs b/DiscordBot/Extensions/DBConnectionExtension.cs index bba946b9..44acc354 100644 --- a/DiscordBot/Extensions/DBConnectionExtension.cs +++ b/DiscordBot/Extensions/DBConnectionExtension.cs @@ -7,8 +7,7 @@ public static class DBConnectionExtension { public static async Task ColumnExists(this DbConnection connection, string tableName, string columnName) { - // Execute the query `SHOW COLUMNS FROM `{tableName}` LIKE '{columnName}'` and check if any rows are returned - var query = $"SHOW COLUMNS FROM `{tableName}` LIKE '{columnName}'"; + var query = $"SELECT 1 FROM information_schema.columns WHERE table_name = '{tableName}' AND column_name = '{columnName}'"; var response = await connection.QuerySqlAsync(query); return response.Count > 0; } diff --git a/DiscordBot/Extensions/UserDBRepository.cs b/DiscordBot/Extensions/UserDBRepository.cs index b5c62837..81dcd90c 100644 --- a/DiscordBot/Extensions/UserDBRepository.cs +++ b/DiscordBot/Extensions/UserDBRepository.cs @@ -23,7 +23,7 @@ public class ServerUser public static class UserProps { public const string TableName = "users"; - + public const string UserID = nameof(ServerUser.UserID); public const string Karma = nameof(ServerUser.Karma); public const string KarmaWeekly = nameof(ServerUser.KarmaWeekly); @@ -38,8 +38,8 @@ public static class UserProps public interface IServerUserRepo { [Sql($@" - INSERT INTO {UserProps.TableName} ({UserProps.UserID}) VALUES (@{UserProps.UserID}); - SELECT * FROM {UserProps.TableName} WHERE {UserProps.UserID} = @{UserProps.UserID}")] + INSERT INTO {UserProps.TableName} ({UserProps.UserID}) VALUES (@{UserProps.UserID}) + RETURNING *")] Task InsertUser(ServerUser user); [Sql($"DELETE FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] Task RemoveUser(string userId); @@ -48,26 +48,26 @@ public interface IServerUserRepo Task GetUser(string userId); #region Ranks - - [Sql($"SELECT {UserProps.UserID}, {UserProps.Karma}, {UserProps.Level}, {UserProps.Exp} FROM {UserProps.TableName} ORDER BY {UserProps.Level} DESC, RAND() LIMIT @n")] + + [Sql($"SELECT {UserProps.UserID}, {UserProps.Karma}, {UserProps.Level}, {UserProps.Exp} FROM {UserProps.TableName} ORDER BY {UserProps.Level} DESC, RANDOM() LIMIT @n")] Task> GetTopLevel(int n); - [Sql($"SELECT {UserProps.UserID}, {UserProps.Karma}, {UserProps.KarmaGiven} FROM {UserProps.TableName} ORDER BY {UserProps.Karma} DESC, RAND() LIMIT @n")] + [Sql($"SELECT {UserProps.UserID}, {UserProps.Karma}, {UserProps.KarmaGiven} FROM {UserProps.TableName} ORDER BY {UserProps.Karma} DESC, RANDOM() LIMIT @n")] Task> GetTopKarma(int n); - [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaWeekly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaWeekly} DESC, RAND() LIMIT @n")] + [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaWeekly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaWeekly} DESC, RANDOM() LIMIT @n")] Task> GetTopKarmaWeekly(int n); - [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaMonthly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaMonthly} DESC, RAND() LIMIT @n")] + [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaMonthly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaMonthly} DESC, RANDOM() LIMIT @n")] Task> GetTopKarmaMonthly(int n); - [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaYearly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaYearly} DESC, RAND() LIMIT @n")] + [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaYearly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaYearly} DESC, RANDOM() LIMIT @n")] Task> GetTopKarmaYearly(int n); [Sql($"SELECT COUNT({UserProps.UserID})+1 FROM {UserProps.TableName} WHERE {UserProps.Level} > @level")] Task GetLevelRank(string userId, uint level); [Sql($"SELECT COUNT({UserProps.UserID})+1 FROM {UserProps.TableName} WHERE {UserProps.Karma} > @karma")] Task GetKarmaRank(string userId, uint karma); - + #endregion // Ranks #region Update Values - + [Sql($"UPDATE {UserProps.TableName} SET {UserProps.Karma} = @karma WHERE {UserProps.UserID} = @userId")] Task UpdateKarma(string userId, uint karma); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.Karma} = {UserProps.Karma} + 1, {UserProps.KarmaWeekly} = {UserProps.KarmaWeekly} + 1, {UserProps.KarmaMonthly} = {UserProps.KarmaMonthly} + 1, {UserProps.KarmaYearly} = {UserProps.KarmaYearly} + 1 WHERE {UserProps.UserID} = @userId")] @@ -80,11 +80,11 @@ public interface IServerUserRepo Task UpdateLevel(string userId, uint level); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.DefaultCity} = @city WHERE {UserProps.UserID} = @userId")] Task UpdateDefaultCity(string userId, string city); - + #endregion // Update Values #region Get Single Values - + [Sql($"SELECT {UserProps.Karma} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] Task GetKarma(string userId); [Sql($"SELECT {UserProps.KarmaGiven} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] @@ -95,7 +95,7 @@ public interface IServerUserRepo Task GetLevel(string userId); [Sql($"SELECT {UserProps.DefaultCity} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] Task GetDefaultCity(string userId); - + #endregion // Get Single Values /// Returns a count of {Props.TableName} in the Table, otherwise it fails. diff --git a/DiscordBot/Modules/ModerationModule.cs b/DiscordBot/Modules/ModerationModule.cs index 48de2952..69c17141 100644 --- a/DiscordBot/Modules/ModerationModule.cs +++ b/DiscordBot/Modules/ModerationModule.cs @@ -7,7 +7,6 @@ using Pathoschild.NaturalTimeParser.Parser; using DiscordBot.Attributes; using DiscordBot.Utils; -using Org.BouncyCastle.Asn1.Cms; namespace DiscordBot.Modules; @@ -22,13 +21,13 @@ public class ModerationModule : ModuleBase public BotSettings Settings { get; set; } public UserService UserService { get; set; } public ModerationService ModerationService { get; set; } - + #endregion - + private async Task IsModerationEnabled() { if (Settings.ModeratorCommandsEnabled) return true; - if (await Context.Guild.GetChannelAsync(Settings.BotAnnouncementChannel.Id)is IMessageChannel botAnnouncementChannel) + if (await Context.Guild.GetChannelAsync(Settings.BotAnnouncementChannel.Id) is IMessageChannel botAnnouncementChannel) { var sentMessage = await botAnnouncementChannel.SendMessageAsync($"{Context.User.Mention} some moderation commands are disabled, try using Wick."); await Context.Message.DeleteAsync(); @@ -114,7 +113,7 @@ await LoggingService.LogChannelAndFile( $"You have been muted from UDC for **{Utils.Utils.FormatTime(seconds)}** for the following reason : **{message}**. " + "This is not appealable and any tentative to avoid it will result in your permanent ban.")) { - if (await Context.Guild.GetChannelAsync(Settings.BotCommandsChannel.Id)is ISocketMessageChannel botCommandChannel) + if (await Context.Guild.GetChannelAsync(Settings.BotCommandsChannel.Id) is ISocketMessageChannel botCommandChannel) await botCommandChannel.SendMessageAsync( $"I could not DM you {user.Mention}!\nYou have been muted from UDC for **{Utils.Utils.FormatTime(seconds)}** for the following reason : **{message}**. " + "This is not appealable and any tentative to avoid it will result in your permanent ban."); @@ -442,7 +441,7 @@ public async Task FullSync() } #region General Utility Commands - + [Command("WelcomeMessageCount")] [Summary("Returns a count of pending welcome messages.")] [RequireModerator, HideFromHelp] @@ -462,7 +461,7 @@ public async Task WelcomeMessageCount() } await Context.Message.DeleteAsync(); } - + // Command to show the tags available for a specific channel, so the command needs to be run in a channel with tags or specific a channel id to check [Command("ChannelTags")] [Summary("Returns a list of tags for the current channel.")] @@ -499,7 +498,7 @@ public async Task ChannelTags(ulong channelId) } #endregion - + #region CommandList [RequireModerator] [Summary("Does what you see now.")] diff --git a/DiscordBot/Program.cs b/DiscordBot/Program.cs index 03989537..f0f22093 100644 --- a/DiscordBot/Program.cs +++ b/DiscordBot/Program.cs @@ -75,6 +75,7 @@ private async Task MainAsync() _recruitService = _services.GetRequiredService(); _services.GetRequiredService(); _services.GetRequiredService(); + _services.GetRequiredService(); return Task.CompletedTask; }; @@ -110,6 +111,7 @@ private IServiceProvider ConfigureServices() => .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .BuildServiceProvider(); private static void DeserializeSettings() diff --git a/DiscordBot/Services/DatabaseService.cs b/DiscordBot/Services/DatabaseService.cs index 82df7e62..11549fe9 100644 --- a/DiscordBot/Services/DatabaseService.cs +++ b/DiscordBot/Services/DatabaseService.cs @@ -3,7 +3,8 @@ using DiscordBot.Domain; using DiscordBot.Settings; using Insight.Database; -using MySql.Data.MySqlClient; +using Insight.Database.Providers.PostgreSQL; +using Npgsql; namespace DiscordBot.Services; @@ -18,7 +19,7 @@ private ICasinoRepo CreateCasinoQuery() { try { - var c = new MySqlConnection(ConnectionString); + var c = new NpgsqlConnection(ConnectionString); return c.As(); } catch (Exception e) @@ -32,7 +33,7 @@ private IServerUserRepo CreateQuery() { try { - var c = new MySqlConnection(ConnectionString); + var c = new NpgsqlConnection(ConnectionString); return c.As(); } catch (Exception e) @@ -47,13 +48,15 @@ private IServerUserRepo CreateQuery() public DatabaseService(ILoggingService logging, BotSettings settings) { + PostgreSQLInsightDbProvider.RegisterProvider(); + ConnectionString = settings.DbConnectionString; _logging = logging; DbConnection c = null; try { - c = new MySqlConnection(ConnectionString); + c = new NpgsqlConnection(ConnectionString); } catch (Exception e) { @@ -72,11 +75,10 @@ await _logging.LogAction( $"{ServiceName}: Connected to database successfully. {userCount} users in database.", ExtendedLogSeverity.Positive); - // Not sure on best practice for if column is missing, full blown migrations seem overkill var defaultCityExists = await c.ColumnExists(UserProps.TableName, UserProps.DefaultCity); if (!defaultCityExists) { - c.ExecuteSql($"ALTER TABLE `{UserProps.TableName}` ADD `{UserProps.DefaultCity}` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL AFTER `{UserProps.Level}`"); + c.ExecuteSql($"ALTER TABLE \"{UserProps.TableName}\" ADD COLUMN \"{UserProps.DefaultCity}\" varchar(64) DEFAULT NULL"); await _logging.LogAction($"DatabaseService: Added missing column '{UserProps.DefaultCity}' to table '{UserProps.TableName}'.", ExtendedLogSeverity.Positive); } @@ -88,23 +90,17 @@ await _logging.LogAction($"DatabaseService: Table '{UserProps.TableName}' does n try { c.ExecuteSql( - $"CREATE TABLE `{UserProps.TableName}` (`ID` int(11) UNSIGNED NOT NULL," + - $"`{UserProps.UserID}` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, " + - $"`{UserProps.Karma}` int(11) UNSIGNED NOT NULL DEFAULT 0, " + - $"`{UserProps.KarmaWeekly}` int(11) UNSIGNED NOT NULL DEFAULT 0, " + - $"`{UserProps.KarmaMonthly}` int(11) UNSIGNED NOT NULL DEFAULT 0, " + - $"`{UserProps.KarmaYearly}` int(11) UNSIGNED NOT NULL DEFAULT 0, " + - $"`{UserProps.KarmaGiven}` int(11) UNSIGNED NOT NULL DEFAULT 0, " + - $"`{UserProps.Exp}` bigint(11) UNSIGNED NOT NULL DEFAULT 0, " + - $"`{UserProps.Level}` int(11) UNSIGNED NOT NULL DEFAULT 0) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); - c.ExecuteSql( - $"ALTER TABLE `{UserProps.TableName}` ADD PRIMARY KEY (`ID`,`{UserProps.UserID}`), ADD UNIQUE KEY `{UserProps.UserID}` (`{UserProps.UserID}`)"); - c.ExecuteSql( - $"ALTER TABLE `{UserProps.TableName}` MODIFY `ID` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, AUTO_INCREMENT=1"); - - // "DefaultCity" Nullable - Weather, BDay, Temp, Time, etc. Optional for users to set their own city (Added - Jan 2024) - c.ExecuteSql( - $"ALTER TABLE `{UserProps.TableName}` ADD `{UserProps.DefaultCity}` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL AFTER `{UserProps.Level}`"); + $"CREATE TABLE \"{UserProps.TableName}\" (" + + $"\"ID\" SERIAL PRIMARY KEY, " + + $"\"{UserProps.UserID}\" varchar(32) NOT NULL UNIQUE, " + + $"\"{UserProps.Karma}\" integer NOT NULL DEFAULT 0, " + + $"\"{UserProps.KarmaWeekly}\" integer NOT NULL DEFAULT 0, " + + $"\"{UserProps.KarmaMonthly}\" integer NOT NULL DEFAULT 0, " + + $"\"{UserProps.KarmaYearly}\" integer NOT NULL DEFAULT 0, " + + $"\"{UserProps.KarmaGiven}\" integer NOT NULL DEFAULT 0, " + + $"\"{UserProps.Exp}\" bigint NOT NULL DEFAULT 0, " + + $"\"{UserProps.Level}\" integer NOT NULL DEFAULT 0, " + + $"\"{UserProps.DefaultCity}\" varchar(64) DEFAULT NULL)"); } catch (Exception e) { @@ -133,31 +129,27 @@ await _logging.LogAction( ExtendedLogSeverity.LowWarning); try { - // Create casino_users table c.ExecuteSql( - $"CREATE TABLE `{CasinoProps.CasinoTableName}` (" + - $"`{CasinoProps.Id}` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, " + - $"`{CasinoProps.UserID}` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, " + - $"`{CasinoProps.Tokens}` bigint(20) UNSIGNED NOT NULL DEFAULT 1000, " + - $"`{CasinoProps.CreatedAt}` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, " + - $"`{CasinoProps.UpdatedAt}` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, " + - $"`{CasinoProps.LastDailyReward}` timestamp NOT NULL DEFAULT '1970-01-01 00:00:01', " + - $"PRIMARY KEY (`{CasinoProps.Id}`), " + - $"UNIQUE KEY `{CasinoProps.UserID}` (`{CasinoProps.UserID}`) " + - $") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); + $"CREATE TABLE \"{CasinoProps.CasinoTableName}\" (" + + $"\"{CasinoProps.Id}\" SERIAL PRIMARY KEY, " + + $"\"{CasinoProps.UserID}\" varchar(32) NOT NULL UNIQUE, " + + $"\"{CasinoProps.Tokens}\" bigint NOT NULL DEFAULT 1000, " + + $"\"{CasinoProps.CreatedAt}\" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, " + + $"\"{CasinoProps.UpdatedAt}\" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, " + + $"\"{CasinoProps.LastDailyReward}\" timestamptz NOT NULL DEFAULT '1970-01-01 00:00:01+00')"); + + c.ExecuteSql( + $"CREATE TABLE \"{CasinoProps.TransactionTableName}\" (" + + $"\"{CasinoProps.TransactionId}\" SERIAL PRIMARY KEY, " + + $"\"{CasinoProps.TransactionUserID}\" varchar(32) NOT NULL, " + + $"\"{CasinoProps.Amount}\" bigint NOT NULL, " + + $"\"{CasinoProps.TransactionType}\" integer NOT NULL, " + + $"\"{CasinoProps.Details}\" jsonb DEFAULT NULL, " + + $"\"{CasinoProps.TransactionCreatedAt}\" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)"); - // Create token_transactions table c.ExecuteSql( - $"CREATE TABLE `{CasinoProps.TransactionTableName}` (" + - $"`{CasinoProps.TransactionId}` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, " + - $"`{CasinoProps.TransactionUserID}` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL, " + - $"`{CasinoProps.Amount}` bigint(20) NOT NULL, " + - $"`{CasinoProps.TransactionType}` int(11) NOT NULL, " + - $"`{CasinoProps.Details}` json DEFAULT NULL, " + // JSON column for transaction details - $"`{CasinoProps.TransactionCreatedAt}` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, " + - $"PRIMARY KEY (`{CasinoProps.TransactionId}`), " + - $"KEY `idx_user_created` (`{CasinoProps.TransactionUserID}`, `{CasinoProps.TransactionCreatedAt}`) " + - $") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); + $"CREATE INDEX idx_user_created ON \"{CasinoProps.TransactionTableName}\" " + + $"(\"{CasinoProps.TransactionUserID}\", \"{CasinoProps.TransactionCreatedAt}\")"); } catch (Exception e) { @@ -171,24 +163,6 @@ await _logging.LogAction($"DatabaseService: Casino tables generated without erro ExtendedLogSeverity.Positive); c.Close(); } - - // Generate and add events if they don't exist - try - { - c.ExecuteSql( - $"CREATE EVENT IF NOT EXISTS `ResetWeeklyLeaderboards` ON SCHEDULE EVERY 1 WEEK STARTS '2021-08-02 00:00:00' ON COMPLETION NOT PRESERVE ENABLE DO UPDATE {c.Database}.users SET {UserProps.KarmaWeekly} = 0"); - c.ExecuteSql( - $"CREATE EVENT IF NOT EXISTS `ResetMonthlyLeaderboards` ON SCHEDULE EVERY 1 MONTH STARTS '2021-08-01 00:00:00' ON COMPLETION NOT PRESERVE ENABLE DO UPDATE {c.Database}.users SET {UserProps.KarmaMonthly} = 0"); - c.ExecuteSql( - $"CREATE EVENT IF NOT EXISTS `ResetYearlyLeaderboards` ON SCHEDULE EVERY 1 YEAR STARTS '2022-01-01 00:00:00' ON COMPLETION NOT PRESERVE ENABLE DO UPDATE {c.Database}.users SET {UserProps.KarmaYearly} = 0"); - c.Close(); - } - catch (Exception e) - { - await _logging.LogAction($"SQL Exception: Failed to generate leaderboard events.\nMessage: {e}", - ExtendedLogSeverity.Warning); - } - }); } diff --git a/DiscordBot/Services/KarmaResetService.cs b/DiscordBot/Services/KarmaResetService.cs new file mode 100644 index 00000000..d47f82e7 --- /dev/null +++ b/DiscordBot/Services/KarmaResetService.cs @@ -0,0 +1,136 @@ +using DiscordBot.Settings; +using Insight.Database; +using Npgsql; + +namespace DiscordBot.Services; + +/// +/// Replaces MySQL EVENT scheduler — resets weekly/monthly/yearly karma columns on schedule. +/// Tracks last-reset timestamps so missed resets are caught up on startup. +/// +public class KarmaResetService +{ + private const string MetaTable = "karma_reset_meta"; + + private readonly ILoggingService _logging; + private readonly string _connectionString; + + public KarmaResetService(ILoggingService logging, BotSettings settings) + { + _logging = logging; + _connectionString = settings.DbConnectionString; + + Task.Run(RunLoop); + } + + private async Task RunLoop() + { + // Wait for DatabaseService to finish table creation + await Task.Delay(TimeSpan.FromSeconds(10)); + + try + { + await EnsureMetaTable(); + await CatchUpMissedResets(); + } + catch (Exception e) + { + await _logging.LogChannelAndFile($"KarmaResetService: Failed during startup: {e.Message}", ExtendedLogSeverity.Warning); + } + + while (true) + { + try + { + await Task.Delay(TimeSpan.FromHours(1)); + + var now = DateTime.UtcNow; + + if (now.DayOfWeek == DayOfWeek.Monday) + await TryReset("weekly", UserProps.KarmaWeekly); + + if (now.Day == 1) + { + await TryReset("monthly", UserProps.KarmaMonthly); + + if (now.Month == 1) + await TryReset("yearly", UserProps.KarmaYearly); + } + } + catch (Exception e) + { + await _logging.LogChannelAndFile($"KarmaResetService: Error during reset check: {e.Message}", ExtendedLogSeverity.Warning); + } + } + } + + private async Task EnsureMetaTable() + { + await using var c = new NpgsqlConnection(_connectionString); + await c.OpenAsync(); + await c.ExecuteSqlAsync( + $"CREATE TABLE IF NOT EXISTS \"{MetaTable}\" (" + + $"\"period\" varchar(16) PRIMARY KEY, " + + $"\"last_reset\" timestamptz NOT NULL DEFAULT '1970-01-01 00:00:00+00')"); + await c.ExecuteSqlAsync($"INSERT INTO \"{MetaTable}\" (\"period\") VALUES ('weekly') ON CONFLICT DO NOTHING"); + await c.ExecuteSqlAsync($"INSERT INTO \"{MetaTable}\" (\"period\") VALUES ('monthly') ON CONFLICT DO NOTHING"); + await c.ExecuteSqlAsync($"INSERT INTO \"{MetaTable}\" (\"period\") VALUES ('yearly') ON CONFLICT DO NOTHING"); + } + + private async Task CatchUpMissedResets() + { + var now = DateTime.UtcNow; + + var weeklyLast = await GetLastReset("weekly"); + if (WeekNumber(now) != WeekNumber(weeklyLast) || now.Year != weeklyLast.Year) + await ResetColumn("weekly", UserProps.KarmaWeekly); + + var monthlyLast = await GetLastReset("monthly"); + if (now.Month != monthlyLast.Month || now.Year != monthlyLast.Year) + await ResetColumn("monthly", UserProps.KarmaMonthly); + + var yearlyLast = await GetLastReset("yearly"); + if (now.Year != yearlyLast.Year) + await ResetColumn("yearly", UserProps.KarmaYearly); + } + + private async Task TryReset(string period, string column) + { + var lastReset = await GetLastReset(period); + var now = DateTime.UtcNow; + + var shouldReset = period switch + { + "weekly" => WeekNumber(now) != WeekNumber(lastReset) || now.Year != lastReset.Year, + "monthly" => now.Month != lastReset.Month || now.Year != lastReset.Year, + "yearly" => now.Year != lastReset.Year, + _ => false + }; + + if (shouldReset) + await ResetColumn(period, column); + } + + private async Task ResetColumn(string period, string column) + { + await using var c = new NpgsqlConnection(_connectionString); + await c.OpenAsync(); + await c.ExecuteSqlAsync($"UPDATE \"{UserProps.TableName}\" SET \"{column}\" = 0"); + await c.ExecuteSqlAsync($"UPDATE \"{MetaTable}\" SET \"last_reset\" = NOW() WHERE \"period\" = '{period}'"); + await _logging.LogChannelAndFile($"KarmaResetService: Reset {period} karma ({column}).", ExtendedLogSeverity.Positive); + } + + private async Task GetLastReset(string period) + { + await using var c = new NpgsqlConnection(_connectionString); + await c.OpenAsync(); + var results = await c.QuerySqlAsync($"SELECT \"last_reset\" FROM \"{MetaTable}\" WHERE \"period\" = '{period}'"); + if (results.Count > 0 && results[0] is IDictionary row && row.TryGetValue("last_reset", out var val) && val is DateTime dt) + return dt; + return DateTime.MinValue; + } + + private static int WeekNumber(DateTime date) => + System.Globalization.CultureInfo.InvariantCulture.Calendar + .GetWeekOfYear(date, System.Globalization.CalendarWeekRule.FirstFourDayWeek, DayOfWeek.Monday); +} \ No newline at end of file diff --git a/DiscordBot/Settings/Settings.example.json b/DiscordBot/Settings/Settings.example.json index 44960198..db00b571 100644 --- a/DiscordBot/Settings/Settings.example.json +++ b/DiscordBot/Settings/Settings.example.json @@ -3,9 +3,8 @@ /* Auth info */ "token": "Y O U R _ B O T _ T O K E N", "invite": "InviteLink", // Currently Unused - /* 'SSL MODE' and 'Allow User Variables' are only required when running on a local machine with XAMPP. This can often be removed. */ /* DB Info*/ - "DbConnectionString": "server=localhost;port=3306;database=test;user id=USER;Password=USERPASSWORD;SSL Mode=None;Allow User Variables=True", + "DbConnectionString": "Host=localhost;Port=5432;Database=udcbot;Username=udcbot;Password=USERPASSWORD", /*Server Info*/ "serverRootPath": "./SERVER", "assetsRootPath": "./Assets", diff --git a/README.md b/README.md index 4052370e..05e18277 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,14 @@ This bot follows a **Service-Module** architecture pattern designed for maintain ### Services vs Modules **Services** (`/DiscordBot/Services/`) contain the core business logic and data operations: + - Handle database interactions, API calls, and background tasks - Maintain state and provide reusable functionality - Examples: `UserService`, `DatabaseService`, `ModerationService`, `LoggingService` - Registered as singletons in the dependency injection container **Modules** (`/DiscordBot/Modules/`) handle Discord command interactions: + - Expose functionality to users via chat commands - Use `[Command]` attributes to define command behavior - Receive services via dependency injection @@ -26,6 +28,7 @@ This bot follows a **Service-Module** architecture pattern designed for maintain ### Dependency Injection The bot uses .NET's built-in dependency injection system: + - Services are registered in `Program.cs` using `ConfigureServices()` - Modules receive services via public property injection - This allows for loose coupling and easier testing @@ -33,6 +36,7 @@ The bot uses .NET's built-in dependency injection system: ### Command System Commands are implemented using Discord.Net's command framework: + - Commands use attributes like `[Command("commandname")]` and `[Summary("description")]` - Custom attributes provide authorization: `[RequireModerator]`, `[RequireAdmin]` - Command routing is handled by `CommandHandlingService` @@ -55,7 +59,8 @@ public async Task MyCommand(string parameter) } ``` -3. **Inject required services** via public properties: +1. **Inject required services** via public properties: + ```csharp public UserService UserService { get; set; } public DatabaseService DatabaseService { get; set; } @@ -64,6 +69,7 @@ public DatabaseService DatabaseService { get; set; } ### Creating a New Service 1. **Create your service class** in `/DiscordBot/Services/`: + ```csharp public class MyNewService { @@ -81,12 +87,14 @@ public class MyNewService } ``` -2. **Register the service** in `Program.cs` within `ConfigureServices()`: +1. **Register the service** in `Program.cs` within `ConfigureServices()`: + ```csharp .AddSingleton() ``` -3. **Inject it into modules** that need it: +1. **Inject it into modules** that need it: + ```csharp public MyNewService MyNewService { get; set; } ``` @@ -126,11 +134,11 @@ public class RequireMyRoleAttribute : PreconditionAttribute - [Compiling](#compiling) - [Dependencies](#dependencies) - [Running](#running) - - [Docker](#docker) - - [Runtime Dependencies](#runtime-dependencies) + - [Docker](#docker) + - [Runtime Dependencies](#runtime-dependencies) - [Notes](#notes) - - [Logging](#logging) - - [Discord.Net](#discordnet) + - [Logging](#logging) + - [Discord.Net](#discordnet) - [FAQ](#faq) ## Compiling @@ -140,13 +148,16 @@ public class RequireMyRoleAttribute : PreconditionAttribute To successfully compile you will need the following: **Required:** + - [.NET 6.0 SDK](https://dotnet.microsoft.com/download/dotnet/6.0) or later - An IDE such as [Visual Studio](https://visualstudio.microsoft.com/vs/community/), [VS Code](https://code.visualstudio.com/), or [JetBrains Rider](https://www.jetbrains.com/rider/) **Recommended for Development:** + - [Docker](https://www.docker.com/get-started) and [Docker Compose](https://docs.docker.com/compose/install/) for database containerization **Build the project:** + ```bash dotnet restore dotnet build @@ -180,6 +191,7 @@ _For production deployment, consult the [Discord.Net Deployment Guide](https://d **Recommended for development:** Docker simplifies database setup and ensures consistency. **To run with Docker:** + ```bash # Start both database and bot docker-compose up @@ -189,11 +201,13 @@ docker-compose up database ``` **Development workflow:** + 1. Start the database container: `docker-compose up database` 2. Update the `DbConnectionString` in `Settings.json` to match your docker-compose configuration 3. Run the bot from your IDE for faster development iteration **Full Docker deployment:** + ```bash # Build and start everything docker-compose up --build --remove-orphans @@ -208,16 +222,17 @@ docker-compose up -d **Manual Database Setup (Alternative to Docker):** -If you prefer not to use Docker, you'll need to set up a MySQL database manually: +If you prefer not to use Docker, you'll need to set up a PostgreSQL database manually: -1. **Install MySQL server:** - - **Windows/macOS:** [XAMPP](https://www.apachefriends.org/download.html) (includes MySQL + phpMyAdmin) - - **Linux:** `sudo apt install mysql-server` or equivalent +1. **Install PostgreSQL:** + - **Windows:** [PostgreSQL Installer](https://www.postgresql.org/download/windows/) + - **macOS:** `brew install postgresql@16` + - **Linux:** `sudo apt install postgresql` or equivalent 2. **Create database and user:** - Create a new database for the bot - Create a user with full permissions to that database - - Update the `DbConnectionString` in `Settings.json` with your database details + - Update the `DbConnectionString` in `Settings.json` with your connection details (e.g. `Host=localhost;Port=5432;Database=udcbot;Username=udcbot;Password=YOUR_PASSWORD`) 3. **Initialize database schema:** - The bot will attempt to create necessary tables on first run @@ -225,11 +240,13 @@ If you prefer not to use Docker, you'll need to set up a MySQL database manually **Additional Linux Requirements:** For image processing functionality, install Microsoft Core Fonts: + ```bash sudo apt install ttf-mscorefonts-installer ``` **Connection String Format:** + ```json "DbConnectionString": "Server=localhost;Database=your_db_name;Uid=your_username;Pwd=your_password;" ``` @@ -241,6 +258,7 @@ sudo apt install ttf-mscorefonts-installer The bot includes comprehensive logging to help with troubleshooting: **Log Levels and Colors:** + - **Critical/Error:** Red text - Something is broken and needs immediate attention - **Warning:** Yellow text - Potential issues that should be investigated - **Info:** White text - General operational information @@ -249,6 +267,7 @@ The bot includes comprehensive logging to help with troubleshooting: **During startup:** Any yellow or red messages likely indicate configuration or connectivity issues. **Log Locations:** + - Console output for immediate feedback - Channel logging (if configured) for persistent records - See [LoggingService](https://github.com/Unity-Developer-Community/UDC-Bot/blob/dev/DiscordBot/Services/LoggingService.cs) for implementation details @@ -258,16 +277,19 @@ The bot includes comprehensive logging to help with troubleshooting: This bot is built on [Discord.Net](https://discordnet.dev/), a powerful .NET library for Discord bots. **Key Concepts to Understand:** + - **Asynchronous Programming:** Extensive use of `async`/`await` patterns - **Event-Driven Architecture:** Reactions to Discord events (messages, user joins, etc.) - **Polymorphism:** Rich type hierarchy for Discord entities (users, channels, guilds) **Helpful Resources:** + - [Discord.Net Documentation](https://discordnet.dev/guides/introduction/intro.html) - [Discord.Net API Reference](https://discordnet.dev/api/index.html) - [Discord Developer Portal](https://discord.com/developers/docs) for Discord API specifics **Common Patterns in this Bot:** + - Commands return `Task` for async operations - Heavy use of dependency injection for service access - Event handlers for background functionality (user joins, message processing) @@ -278,6 +300,7 @@ This bot is built on [Discord.Net](https://discordnet.dev/), a powerful .NET lib **Q: The bot won't start - what should I check?** A: Verify these in order: + 1. Bot token is correctly set in `Settings.json` 2. Database connection string is correct and database is accessible 3. All required folders (SERVER, Settings) are in the right location @@ -286,6 +309,7 @@ A: Verify these in order: **Q: "Unable to load the service index" or NuGet restore errors** A: This is usually a temporary network issue with package sources. Try: **Warning:** Clearing NuGet locals will remove all cached packages and temporary files. This may require re-downloading dependencies, which could take significant time on slower connections. + ```bash dotnet nuget locals all --clear dotnet restore @@ -293,13 +317,15 @@ dotnet restore **Q: Database connection fails** A: Common causes: + - Incorrect connection string format - Database server not running - User permissions insufficient - Firewall blocking database port **Q: How do I get a Discord bot token?** -A: +A: + 1. Go to [Discord Developer Portal](https://discord.com/developers/applications) 2. Create a new application 3. Go to "Bot" section @@ -308,6 +334,7 @@ A: **Q: What permissions does the bot need?** A: The bot requires: + - Read Messages - Send Messages - Manage Messages (for moderation features) @@ -316,7 +343,8 @@ A: The bot requires: - Additional permissions based on enabled features **Q: How can I contribute or report bugs?** -A: +A: + - Check existing issues on GitHub - For bugs: provide console logs and steps to reproduce - For contributions: see the [Contributing](#contributing) section above @@ -324,13 +352,15 @@ A: ### Development Tips **Q: How do I debug commands?** -A: +A: + - Use the logging system: `LoggingService.LogToConsole(message, ExtendedLogSeverity.Info)` - Set breakpoints in your IDE when running the bot locally - Check the command history in `CommandHandlingService` **Q: My command isn't working** A: Common issues: + - Missing `[Command]` attribute - Incorrect parameter types - Missing dependency injection setup diff --git a/docker-compose.yml b/docker-compose.yml index 0c4aba68..77740b80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,28 +1,27 @@ version: "3.3" services: db: - image: mysql + image: postgres:16 volumes: - - db_data:/var/lib/mysql + - db_data:/var/lib/postgresql/data restart: always ports: - - 127.0.0.1:3306:3306 - #- 3306:3306 + - 127.0.0.1:5432:5432 environment: - MYSQL_ROOT_PASSWORD: 123456789 - MYSQL_DATABASE: udcbot - MYSQL_USER: udcbot - MYSQL_PASSWORD: 123456789 + POSTGRES_DB: udcbot + POSTGRES_USER: udcbot + POSTGRES_PASSWORD: 123456789 - phpmyadmin: - image: phpmyadmin + pgadmin: + image: dpage/pgadmin4 depends_on: - db restart: always ports: - 8080:80 environment: - PMA_HOST: db + PGADMIN_DEFAULT_EMAIL: admin@local.dev + PGADMIN_DEFAULT_PASSWORD: admin bot: build: . diff --git a/docs/plans/done/mysql-to-postgresql-migration.md b/docs/plans/done/mysql-to-postgresql-migration.md new file mode 100644 index 00000000..f1b8e7ab --- /dev/null +++ b/docs/plans/done/mysql-to-postgresql-migration.md @@ -0,0 +1,49 @@ +# MySQL → PostgreSQL Migration + +**Status:** Done +**Date:** 2025-07-15 + +## Summary + +Migrated the entire data layer and infrastructure from MySQL 8.0 to PostgreSQL 16. + +## Changes + +### Application Code + +- **NuGet:** Removed `MySql.Data`, added `Insight.Database.Providers.PostgreSQL` (brings Npgsql transitively) +- **DatabaseService:** `MySqlConnection` → `NpgsqlConnection`, DDL rewritten to PostgreSQL syntax, registered `PostgreSQLInsightDbProvider` +- **DBConnectionExtension:** `SHOW COLUMNS` → `information_schema` query +- **UserDBRepository:** `RAND()` → `RANDOM()`, `INSERT...SELECT` → `INSERT...RETURNING *` +- **CasinoRepository:** `LAST_INSERT_ID()` → `RETURNING *`, added `::jsonb` cast +- **KarmaResetService (new):** Replaces MySQL EVENT scheduler with C# polling loop for weekly/monthly/yearly karma resets +- **Program.cs:** Registered `KarmaResetService` +- **ModerationModule:** Removed orphaned `BouncyCastle` import + +### Infrastructure + +- **docker-compose.yml:** MySQL → postgres:16, phpMyAdmin → pgAdmin4 +- **K8s manifests (dev + prod):** Renamed and rewrote mysql.yaml → postgresql.yaml, mysql-backup.yaml → postgresql-backup.yaml, phpmyadmin.yaml → pgadmin.yaml +- **K8s bot.yaml:** Init container updated to wait for PostgreSQL on port 5432 +- **K8s external-secrets.yaml:** MySQL secrets → `postgresql-credentials` + `pgadmin-credentials` +- **K8s bot-config.yaml:** PostgreSQL connection string format +- **Settings.example.json:** Updated connection string and removed XAMPP comment + +### Documentation + +- **README.md:** Updated manual database setup instructions for PostgreSQL + +## Checklist + +- [x] NuGet packages updated +- [x] All SQL queries ported to PostgreSQL syntax +- [x] MySQL EVENT scheduler replaced with KarmaResetService +- [x] Docker Compose updated +- [x] K8s manifests updated and renamed (dev + prod) +- [x] Settings and connection strings updated +- [x] Build verified (0 errors) +- [x] No remaining MySQL references (except historical comment in KarmaResetService) + +## Data Migration Note + +Existing MySQL data needs to be manually exported and imported into PostgreSQL using `pg_dump`/`pg_restore` or a migration tool like `pgloader`. diff --git a/k8s/dev/bot-config.yaml b/k8s/dev/bot-config.yaml index db1d6292..e702a26a 100644 --- a/k8s/dev/bot-config.yaml +++ b/k8s/dev/bot-config.yaml @@ -13,7 +13,7 @@ data: Settings.json: | { "token": "${BOT_TOKEN}", - "DbConnectionString": "server=mysql;port=3306;database=udcbot;user id=udcbot;Password=${DB_PASSWORD};SSL Mode=None;Allow User Variables=True;AllowPublicKeyRetrieval=True", + "DbConnectionString": "Host=postgresql;Port=5432;Database=udcbot;Username=udcbot;Password=${DB_PASSWORD}", "invite": "InviteLink", // Currently Unused /*Server Info*/ "serverRootPath": "./SERVER", diff --git a/k8s/dev/bot.yaml b/k8s/dev/bot.yaml index f3d2a97c..4f88c6ef 100644 --- a/k8s/dev/bot.yaml +++ b/k8s/dev/bot.yaml @@ -60,7 +60,7 @@ spec: - name: DB_PASSWORD valueFrom: secretKeyRef: - name: mysql-user-credentials + name: postgresql-credentials key: password - name: WEATHER_KEY valueFrom: @@ -107,14 +107,14 @@ spec: capabilities: drop: - ALL - - name: wait-for-mysql + - name: wait-for-postgresql image: busybox:1.37 command: - sh - -c - | - until nc -z mysql 3306; do - echo "Waiting for MySQL..." + until nc -z postgresql 5432; do + echo "Waiting for PostgreSQL..." sleep 2 done securityContext: diff --git a/k8s/dev/external-secrets.yaml b/k8s/dev/external-secrets.yaml index 97bfd10a..5dbde529 100644 --- a/k8s/dev/external-secrets.yaml +++ b/k8s/dev/external-secrets.yaml @@ -1,9 +1,9 @@ --- -# MySQL root password — from 1Password "MySQL Server - Root User - Dev" +# PostgreSQL password — from 1Password "PostgreSQL Server - Dev" apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: mysql-credentials + name: postgresql-credentials namespace: udc-bot-dev spec: refreshInterval: 1h @@ -11,30 +11,11 @@ spec: name: onepassword kind: ClusterSecretStore target: - name: mysql-credentials + name: postgresql-credentials data: - secretKey: password remoteRef: - key: "MySQL Server - Root User - Dev" - property: password ---- -# MySQL udcbot user password — from 1Password "MySQL Server - UDC User - Dev" -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: mysql-user-credentials - namespace: udc-bot-dev -spec: - refreshInterval: 1h - secretStoreRef: - name: onepassword - kind: ClusterSecretStore - target: - name: mysql-user-credentials - data: - - secretKey: password - remoteRef: - key: "MySQL Server - UDC User - Dev" + key: "PostgreSQL Server - Dev" property: password --- # Discord bot token — from 1Password "Bot Token - Dev" @@ -56,11 +37,11 @@ spec: key: "Bot Token - Dev" property: identifiant --- -# AWS credentials for MySQL backups to S3 — from 1Password "AWS Backup Credentials" +# pgAdmin login password — from 1Password "pgAdmin - Dev" apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: mysql-backup-credentials + name: pgadmin-credentials namespace: udc-bot-dev spec: refreshInterval: 1h @@ -68,16 +49,12 @@ spec: name: onepassword kind: ClusterSecretStore target: - name: mysql-backup-credentials + name: pgadmin-credentials data: - - secretKey: AWS_ACCESS_KEY_ID - remoteRef: - key: "AWS Backup Credentials" - property: access-key-id - - secretKey: AWS_SECRET_ACCESS_KEY + - secretKey: password remoteRef: - key: "AWS Backup Credentials" - property: secret-access-key + key: "pgAdmin - Dev" + property: password --- # Bot API keys — from various 1Password items apiVersion: external-secrets.io/v1 diff --git a/k8s/dev/mysql-backup.yaml b/k8s/dev/mysql-backup.yaml index ad305e02..b4859d90 100644 --- a/k8s/dev/mysql-backup.yaml +++ b/k8s/dev/mysql-backup.yaml @@ -1,12 +1,12 @@ -# MySQL backup using databack/mysql-backup — handles its own daily schedule internally. -# Dumps go to S3 bucket "udc-bot-mysql-backups" with 90-day retention via S3 lifecycle. +# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. +# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. apiVersion: apps/v1 kind: Deployment metadata: - name: mysql-backup + name: postgresql-backup namespace: udc-bot-dev labels: - app.kubernetes.io/name: mysql-backup + app.kubernetes.io/name: postgresql-backup app.kubernetes.io/part-of: udc-bot environment: dev spec: @@ -15,46 +15,40 @@ spec: type: Recreate selector: matchLabels: - app.kubernetes.io/name: mysql-backup + app.kubernetes.io/name: postgresql-backup template: metadata: labels: - app.kubernetes.io/name: mysql-backup + app.kubernetes.io/name: postgresql-backup app.kubernetes.io/part-of: udc-bot environment: dev spec: containers: - - name: mysql-backup - image: databack/mysql-backup:1.4.0 - args: ["dump"] + - name: postgresql-backup + image: prodrigestivill/postgres-backup-local:16 env: - - name: DB_SERVER - value: mysql - - name: DB_PORT - value: "3306" - - name: DB_USER - value: root - - name: DB_PASS + - name: POSTGRES_HOST + value: postgresql + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DB + value: udcbot + - name: POSTGRES_USER + value: udcbot + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: mysql-credentials + name: postgresql-credentials key: password - - name: DB_DUMP_TARGET - value: s3://udc-bot-mysql-backups/udc-bot-mysql-dev - - name: DB_DUMP_FREQ - value: "1440" - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: mysql-backup-credentials - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: mysql-backup-credentials - key: AWS_SECRET_ACCESS_KEY - - name: AWS_DEFAULT_REGION - value: eu-west-3 + - name: SCHEDULE + value: "@daily" + - name: BACKUP_KEEP_DAYS + value: "90" + - name: BACKUP_DIR + value: /backups + volumeMounts: + - name: backup-data + mountPath: /backups resources: requests: cpu: 25m @@ -67,3 +61,6 @@ spec: capabilities: drop: - ALL + volumes: + - name: backup-data + emptyDir: {} diff --git a/k8s/dev/mysql.yaml b/k8s/dev/mysql.yaml index 8198f72f..33a9eba9 100644 --- a/k8s/dev/mysql.yaml +++ b/k8s/dev/mysql.yaml @@ -2,10 +2,10 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: mysql-data + name: postgresql-data namespace: udc-bot-dev labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: dev spec: @@ -18,10 +18,10 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: mysql + name: postgresql namespace: udc-bot-dev labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: dev spec: @@ -30,38 +30,33 @@ spec: type: Recreate selector: matchLabels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql template: metadata: labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: dev spec: containers: - - name: mysql - image: mysql:8.0 + - name: postgresql + image: postgres:16 ports: - - containerPort: 3306 + - containerPort: 5432 protocol: TCP env: - - name: MYSQL_ROOT_PASSWORD + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: mysql-credentials + name: postgresql-credentials key: password - - name: MYSQL_DATABASE + - name: POSTGRES_DB value: udcbot - - name: MYSQL_USER + - name: POSTGRES_USER value: udcbot - - name: MYSQL_PASSWORD - valueFrom: - secretKeyRef: - name: mysql-user-credentials - key: password volumeMounts: - - name: mysql-data - mountPath: /var/lib/mysql + - name: postgresql-data + mountPath: /var/lib/postgresql/data resources: requests: cpu: 100m @@ -72,9 +67,9 @@ spec: readinessProbe: exec: command: - - sh - - -c - - mysqladmin ping -h 127.0.0.1 -u root -p"${MYSQL_ROOT_PASSWORD}" + - pg_isready + - -U + - udcbot initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 @@ -82,9 +77,9 @@ spec: livenessProbe: exec: command: - - sh - - -c - - mysqladmin ping -h 127.0.0.1 -u root -p"${MYSQL_ROOT_PASSWORD}" + - pg_isready + - -U + - udcbot initialDelaySeconds: 30 periodSeconds: 15 timeoutSeconds: 5 @@ -92,24 +87,24 @@ spec: securityContext: allowPrivilegeEscalation: false volumes: - - name: mysql-data + - name: postgresql-data persistentVolumeClaim: - claimName: mysql-data + claimName: postgresql-data --- apiVersion: v1 kind: Service metadata: - name: mysql + name: postgresql namespace: udc-bot-dev labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: dev spec: type: ClusterIP ports: - - port: 3306 - targetPort: 3306 + - port: 5432 + targetPort: 5432 protocol: TCP selector: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql diff --git a/k8s/dev/pgadmin.yaml b/k8s/dev/pgadmin.yaml new file mode 100644 index 00000000..f6c01f71 --- /dev/null +++ b/k8s/dev/pgadmin.yaml @@ -0,0 +1,99 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgadmin + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pgadmin + template: + metadata: + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: dev + spec: + containers: + - name: pgadmin + image: dpage/pgadmin4:8 + ports: + - containerPort: 80 + protocol: TCP + env: + - name: PGADMIN_DEFAULT_EMAIL + value: admin@udc.ovh + - name: PGADMIN_DEFAULT_PASSWORD + valueFrom: + secretKeyRef: + name: pgadmin-credentials + key: password + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + readinessProbe: + httpGet: + path: /misc/ping + port: 80 + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + name: pgadmin + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 80 + protocol: TCP + selector: + app.kubernetes.io/name: pgadmin +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pgadmin + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: dev + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - pgadmin.dev.bot.udc.ovh + secretName: pgadmin-dev-tls + rules: + - host: pgadmin.dev.bot.udc.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: pgadmin + port: + number: 80 diff --git a/k8s/dev/phpmyadmin.yaml b/k8s/dev/phpmyadmin.yaml index 9f3e216a..f6c01f71 100644 --- a/k8s/dev/phpmyadmin.yaml +++ b/k8s/dev/phpmyadmin.yaml @@ -2,45 +2,48 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: phpmyadmin + name: pgadmin namespace: udc-bot-dev labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: dev spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin template: metadata: labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: dev spec: containers: - - name: phpmyadmin - image: phpmyadmin:5.2.3 + - name: pgadmin + image: dpage/pgadmin4:8 ports: - containerPort: 80 protocol: TCP env: - - name: PMA_HOST - value: mysql - - name: PMA_PORT - value: "3306" + - name: PGADMIN_DEFAULT_EMAIL + value: admin@udc.ovh + - name: PGADMIN_DEFAULT_PASSWORD + valueFrom: + secretKeyRef: + name: pgadmin-credentials + key: password resources: requests: cpu: 25m - memory: 64Mi + memory: 128Mi limits: cpu: 200m memory: 256Mi readinessProbe: httpGet: - path: / + path: /misc/ping port: 80 initialDelaySeconds: 10 periodSeconds: 10 @@ -50,10 +53,10 @@ spec: apiVersion: v1 kind: Service metadata: - name: phpmyadmin + name: pgadmin namespace: udc-bot-dev labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: dev spec: @@ -63,15 +66,15 @@ spec: targetPort: 80 protocol: TCP selector: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: phpmyadmin + name: pgadmin namespace: udc-bot-dev labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: dev annotations: @@ -81,16 +84,16 @@ spec: ingressClassName: traefik tls: - hosts: - - phpmyadmin.dev.bot.udc.ovh - secretName: phpmyadmin-dev-tls + - pgadmin.dev.bot.udc.ovh + secretName: pgadmin-dev-tls rules: - - host: phpmyadmin.dev.bot.udc.ovh + - host: pgadmin.dev.bot.udc.ovh http: paths: - path: / pathType: Prefix backend: service: - name: phpmyadmin + name: pgadmin port: number: 80 diff --git a/k8s/dev/postgresql-backup.yaml b/k8s/dev/postgresql-backup.yaml new file mode 100644 index 00000000..b4859d90 --- /dev/null +++ b/k8s/dev/postgresql-backup.yaml @@ -0,0 +1,66 @@ +# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. +# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql-backup + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: postgresql-backup + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: postgresql-backup + template: + metadata: + labels: + app.kubernetes.io/name: postgresql-backup + app.kubernetes.io/part-of: udc-bot + environment: dev + spec: + containers: + - name: postgresql-backup + image: prodrigestivill/postgres-backup-local:16 + env: + - name: POSTGRES_HOST + value: postgresql + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DB + value: udcbot + - name: POSTGRES_USER + value: udcbot + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql-credentials + key: password + - name: SCHEDULE + value: "@daily" + - name: BACKUP_KEEP_DAYS + value: "90" + - name: BACKUP_DIR + value: /backups + volumeMounts: + - name: backup-data + mountPath: /backups + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumes: + - name: backup-data + emptyDir: {} diff --git a/k8s/dev/postgresql.yaml b/k8s/dev/postgresql.yaml new file mode 100644 index 00000000..33a9eba9 --- /dev/null +++ b/k8s/dev/postgresql.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgresql-data + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: postgresql + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: dev + spec: + containers: + - name: postgresql + image: postgres:16 + ports: + - containerPort: 5432 + protocol: TCP + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql-credentials + key: password + - name: POSTGRES_DB + value: udcbot + - name: POSTGRES_USER + value: udcbot + volumeMounts: + - name: postgresql-data + mountPath: /var/lib/postgresql/data + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + exec: + command: + - pg_isready + - -U + - udcbot + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + exec: + command: + - pg_isready + - -U + - udcbot + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: postgresql-data + persistentVolumeClaim: + claimName: postgresql-data +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP + selector: + app.kubernetes.io/name: postgresql diff --git a/k8s/prod/bot-config.yaml b/k8s/prod/bot-config.yaml index 532b73d7..5354d590 100644 --- a/k8s/prod/bot-config.yaml +++ b/k8s/prod/bot-config.yaml @@ -13,9 +13,8 @@ data: Settings.json: | { "token": "${BOT_TOKEN}", - "DbConnectionString": "server=mysql;port=3306;database=udcbot;user id=udcbot;Password=${DB_PASSWORD};SSL Mode=None;Allow User Variables=True;AllowPublicKeyRetrieval=True", + "DbConnectionString": "Host=postgresql;Port=5432;Database=udcbot;Username=udcbot;Password=${DB_PASSWORD}", "invite": "https://discord.gg/bu3bbby", // Currently Unused - /* 'SSL MODE' and 'Allow User Variables' are only required when running on a local machine with XAMPP. This can often be removed. */ /* DB Info*/ /*Server Info*/ "serverRootPath": "./SERVER", diff --git a/k8s/prod/bot.yaml b/k8s/prod/bot.yaml index 8421fb2d..d6308361 100644 --- a/k8s/prod/bot.yaml +++ b/k8s/prod/bot.yaml @@ -60,7 +60,7 @@ spec: - name: DB_PASSWORD valueFrom: secretKeyRef: - name: mysql-user-credentials + name: postgresql-credentials key: password - name: WEATHER_KEY valueFrom: @@ -107,14 +107,14 @@ spec: capabilities: drop: - ALL - - name: wait-for-mysql + - name: wait-for-postgresql image: busybox:1.37 command: - sh - -c - | - until nc -z mysql 3306; do - echo "Waiting for MySQL..." + until nc -z postgresql 5432; do + echo "Waiting for PostgreSQL..." sleep 2 done securityContext: diff --git a/k8s/prod/external-secrets.yaml b/k8s/prod/external-secrets.yaml index 4b3baa74..f4db19db 100644 --- a/k8s/prod/external-secrets.yaml +++ b/k8s/prod/external-secrets.yaml @@ -1,9 +1,9 @@ --- -# MySQL root password — from 1Password "MySQL Server - Root User - Prod" +# PostgreSQL password — from 1Password "PostgreSQL Server - Prod" apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: mysql-credentials + name: postgresql-credentials namespace: udc-bot-prod spec: refreshInterval: 1h @@ -11,30 +11,11 @@ spec: name: onepassword kind: ClusterSecretStore target: - name: mysql-credentials + name: postgresql-credentials data: - secretKey: password remoteRef: - key: "MySQL Server - Root User - Prod" - property: password ---- -# MySQL udcbot user password — from 1Password "MySQL Server - UDC User - Prod" -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: mysql-user-credentials - namespace: udc-bot-prod -spec: - refreshInterval: 1h - secretStoreRef: - name: onepassword - kind: ClusterSecretStore - target: - name: mysql-user-credentials - data: - - secretKey: password - remoteRef: - key: "MySQL Server - UDC User - Prod" + key: "PostgreSQL Server - Prod" property: password --- # Discord bot token — from 1Password "Bot Token - Prod" @@ -56,11 +37,11 @@ spec: key: "Bot Token - Prod" property: identifiant --- -# AWS credentials for MySQL backups to S3 (shared with dev) +# pgAdmin login password — from 1Password "pgAdmin - Prod" apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: mysql-backup-credentials + name: pgadmin-credentials namespace: udc-bot-prod spec: refreshInterval: 1h @@ -68,16 +49,12 @@ spec: name: onepassword kind: ClusterSecretStore target: - name: mysql-backup-credentials + name: pgadmin-credentials data: - - secretKey: AWS_ACCESS_KEY_ID - remoteRef: - key: "AWS Backup Credentials" - property: access-key-id - - secretKey: AWS_SECRET_ACCESS_KEY + - secretKey: password remoteRef: - key: "AWS Backup Credentials" - property: secret-access-key + key: "pgAdmin - Prod" + property: password --- # Bot API keys — same keys shared between dev and prod apiVersion: external-secrets.io/v1 diff --git a/k8s/prod/mysql-backup.yaml b/k8s/prod/mysql-backup.yaml index 0bd1e584..91f77069 100644 --- a/k8s/prod/mysql-backup.yaml +++ b/k8s/prod/mysql-backup.yaml @@ -1,12 +1,12 @@ -# MySQL backup using databack/mysql-backup — handles its own daily schedule internally. -# Dumps go to S3 bucket "udc-bot-mysql-backups" with 90-day retention via S3 lifecycle. +# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. +# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. apiVersion: apps/v1 kind: Deployment metadata: - name: mysql-backup + name: postgresql-backup namespace: udc-bot-prod labels: - app.kubernetes.io/name: mysql-backup + app.kubernetes.io/name: postgresql-backup app.kubernetes.io/part-of: udc-bot environment: prod spec: @@ -15,46 +15,40 @@ spec: type: Recreate selector: matchLabels: - app.kubernetes.io/name: mysql-backup + app.kubernetes.io/name: postgresql-backup template: metadata: labels: - app.kubernetes.io/name: mysql-backup + app.kubernetes.io/name: postgresql-backup app.kubernetes.io/part-of: udc-bot environment: prod spec: containers: - - name: mysql-backup - image: databack/mysql-backup:1.4.0 - args: ["dump"] + - name: postgresql-backup + image: prodrigestivill/postgres-backup-local:16 env: - - name: DB_SERVER - value: mysql - - name: DB_PORT - value: "3306" - - name: DB_USER - value: root - - name: DB_PASS + - name: POSTGRES_HOST + value: postgresql + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DB + value: udcbot + - name: POSTGRES_USER + value: udcbot + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: mysql-credentials + name: postgresql-credentials key: password - - name: DB_DUMP_TARGET - value: s3://udc-bot-mysql-backups/udc-bot-mysql-prod - - name: DB_DUMP_FREQ - value: "1440" - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: mysql-backup-credentials - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: mysql-backup-credentials - key: AWS_SECRET_ACCESS_KEY - - name: AWS_DEFAULT_REGION - value: eu-west-3 + - name: SCHEDULE + value: "@daily" + - name: BACKUP_KEEP_DAYS + value: "90" + - name: BACKUP_DIR + value: /backups + volumeMounts: + - name: backup-data + mountPath: /backups resources: requests: cpu: 25m @@ -67,3 +61,6 @@ spec: capabilities: drop: - ALL + volumes: + - name: backup-data + emptyDir: {} diff --git a/k8s/prod/mysql.yaml b/k8s/prod/mysql.yaml index 33b73f77..3bcd5dcd 100644 --- a/k8s/prod/mysql.yaml +++ b/k8s/prod/mysql.yaml @@ -2,10 +2,10 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: mysql-data + name: postgresql-data namespace: udc-bot-prod labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: prod spec: @@ -18,10 +18,10 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: mysql + name: postgresql namespace: udc-bot-prod labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: prod spec: @@ -30,38 +30,33 @@ spec: type: Recreate selector: matchLabels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql template: metadata: labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: prod spec: containers: - - name: mysql - image: mysql:8.0 + - name: postgresql + image: postgres:16 ports: - - containerPort: 3306 + - containerPort: 5432 protocol: TCP env: - - name: MYSQL_ROOT_PASSWORD + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: mysql-credentials + name: postgresql-credentials key: password - - name: MYSQL_DATABASE + - name: POSTGRES_DB value: udcbot - - name: MYSQL_USER + - name: POSTGRES_USER value: udcbot - - name: MYSQL_PASSWORD - valueFrom: - secretKeyRef: - name: mysql-user-credentials - key: password volumeMounts: - - name: mysql-data - mountPath: /var/lib/mysql + - name: postgresql-data + mountPath: /var/lib/postgresql/data resources: requests: cpu: 200m @@ -72,9 +67,9 @@ spec: readinessProbe: exec: command: - - sh - - -c - - mysqladmin ping -h 127.0.0.1 -u root -p"${MYSQL_ROOT_PASSWORD}" + - pg_isready + - -U + - udcbot initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 @@ -82,9 +77,9 @@ spec: livenessProbe: exec: command: - - sh - - -c - - mysqladmin ping -h 127.0.0.1 -u root -p"${MYSQL_ROOT_PASSWORD}" + - pg_isready + - -U + - udcbot initialDelaySeconds: 30 periodSeconds: 15 timeoutSeconds: 5 @@ -92,24 +87,24 @@ spec: securityContext: allowPrivilegeEscalation: false volumes: - - name: mysql-data + - name: postgresql-data persistentVolumeClaim: - claimName: mysql-data + claimName: postgresql-data --- apiVersion: v1 kind: Service metadata: - name: mysql + name: postgresql namespace: udc-bot-prod labels: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql app.kubernetes.io/part-of: udc-bot environment: prod spec: type: ClusterIP ports: - - port: 3306 - targetPort: 3306 + - port: 5432 + targetPort: 5432 protocol: TCP selector: - app.kubernetes.io/name: mysql + app.kubernetes.io/name: postgresql diff --git a/k8s/prod/pgadmin.yaml b/k8s/prod/pgadmin.yaml new file mode 100644 index 00000000..ac8b04cb --- /dev/null +++ b/k8s/prod/pgadmin.yaml @@ -0,0 +1,99 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgadmin + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pgadmin + template: + metadata: + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: prod + spec: + containers: + - name: pgadmin + image: dpage/pgadmin4:8 + ports: + - containerPort: 80 + protocol: TCP + env: + - name: PGADMIN_DEFAULT_EMAIL + value: admin@udc.ovh + - name: PGADMIN_DEFAULT_PASSWORD + valueFrom: + secretKeyRef: + name: pgadmin-credentials + key: password + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + readinessProbe: + httpGet: + path: /misc/ping + port: 80 + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + name: pgadmin + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 80 + protocol: TCP + selector: + app.kubernetes.io/name: pgadmin +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pgadmin + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: prod + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - pgadmin.bot.udc.ovh + secretName: pgadmin-prod-tls + rules: + - host: pgadmin.bot.udc.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: pgadmin + port: + number: 80 diff --git a/k8s/prod/phpmyadmin.yaml b/k8s/prod/phpmyadmin.yaml index 4194b194..ac8b04cb 100644 --- a/k8s/prod/phpmyadmin.yaml +++ b/k8s/prod/phpmyadmin.yaml @@ -2,45 +2,48 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: phpmyadmin + name: pgadmin namespace: udc-bot-prod labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: prod spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin template: metadata: labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: prod spec: containers: - - name: phpmyadmin - image: phpmyadmin:5.2.3 + - name: pgadmin + image: dpage/pgadmin4:8 ports: - containerPort: 80 protocol: TCP env: - - name: PMA_HOST - value: mysql - - name: PMA_PORT - value: "3306" + - name: PGADMIN_DEFAULT_EMAIL + value: admin@udc.ovh + - name: PGADMIN_DEFAULT_PASSWORD + valueFrom: + secretKeyRef: + name: pgadmin-credentials + key: password resources: requests: cpu: 25m - memory: 64Mi + memory: 128Mi limits: cpu: 200m memory: 256Mi readinessProbe: httpGet: - path: / + path: /misc/ping port: 80 initialDelaySeconds: 10 periodSeconds: 10 @@ -50,10 +53,10 @@ spec: apiVersion: v1 kind: Service metadata: - name: phpmyadmin + name: pgadmin namespace: udc-bot-prod labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: prod spec: @@ -63,15 +66,15 @@ spec: targetPort: 80 protocol: TCP selector: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: phpmyadmin + name: pgadmin namespace: udc-bot-prod labels: - app.kubernetes.io/name: phpmyadmin + app.kubernetes.io/name: pgadmin app.kubernetes.io/part-of: udc-bot environment: prod annotations: @@ -81,16 +84,16 @@ spec: ingressClassName: traefik tls: - hosts: - - phpmyadmin.bot.udc.ovh - secretName: phpmyadmin-prod-tls + - pgadmin.bot.udc.ovh + secretName: pgadmin-prod-tls rules: - - host: phpmyadmin.bot.udc.ovh + - host: pgadmin.bot.udc.ovh http: paths: - path: / pathType: Prefix backend: service: - name: phpmyadmin + name: pgadmin port: number: 80 diff --git a/k8s/prod/postgresql-backup.yaml b/k8s/prod/postgresql-backup.yaml new file mode 100644 index 00000000..91f77069 --- /dev/null +++ b/k8s/prod/postgresql-backup.yaml @@ -0,0 +1,66 @@ +# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. +# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql-backup + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: postgresql-backup + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: postgresql-backup + template: + metadata: + labels: + app.kubernetes.io/name: postgresql-backup + app.kubernetes.io/part-of: udc-bot + environment: prod + spec: + containers: + - name: postgresql-backup + image: prodrigestivill/postgres-backup-local:16 + env: + - name: POSTGRES_HOST + value: postgresql + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DB + value: udcbot + - name: POSTGRES_USER + value: udcbot + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql-credentials + key: password + - name: SCHEDULE + value: "@daily" + - name: BACKUP_KEEP_DAYS + value: "90" + - name: BACKUP_DIR + value: /backups + volumeMounts: + - name: backup-data + mountPath: /backups + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumes: + - name: backup-data + emptyDir: {} diff --git a/k8s/prod/postgresql.yaml b/k8s/prod/postgresql.yaml new file mode 100644 index 00000000..3bcd5dcd --- /dev/null +++ b/k8s/prod/postgresql.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgresql-data + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: postgresql + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: prod + spec: + containers: + - name: postgresql + image: postgres:16 + ports: + - containerPort: 5432 + protocol: TCP + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql-credentials + key: password + - name: POSTGRES_DB + value: udcbot + - name: POSTGRES_USER + value: udcbot + volumeMounts: + - name: postgresql-data + mountPath: /var/lib/postgresql/data + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + readinessProbe: + exec: + command: + - pg_isready + - -U + - udcbot + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + exec: + command: + - pg_isready + - -U + - udcbot + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: postgresql-data + persistentVolumeClaim: + claimName: postgresql-data +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP + selector: + app.kubernetes.io/name: postgresql From a9f19152df030de4f14d20cf9a645af008370f37 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Wed, 1 Apr 2026 04:06:42 +0200 Subject: [PATCH 02/25] chore(k8s): remove MySQL configurations and update PostgreSQL images --- k8s/dev/mysql-backup.yaml | 66 ------------------- k8s/dev/mysql.yaml | 110 -------------------------------- k8s/dev/pgadmin.yaml | 2 +- k8s/dev/phpmyadmin.yaml | 99 ---------------------------- k8s/dev/postgresql-backup.yaml | 2 +- k8s/dev/postgresql.yaml | 4 +- k8s/prod/mysql-backup.yaml | 66 ------------------- k8s/prod/mysql.yaml | 110 -------------------------------- k8s/prod/pgadmin.yaml | 2 +- k8s/prod/phpmyadmin.yaml | 99 ---------------------------- k8s/prod/postgresql-backup.yaml | 2 +- k8s/prod/postgresql.yaml | 4 +- 12 files changed, 10 insertions(+), 556 deletions(-) delete mode 100644 k8s/dev/mysql-backup.yaml delete mode 100644 k8s/dev/mysql.yaml delete mode 100644 k8s/dev/phpmyadmin.yaml delete mode 100644 k8s/prod/mysql-backup.yaml delete mode 100644 k8s/prod/mysql.yaml delete mode 100644 k8s/prod/phpmyadmin.yaml diff --git a/k8s/dev/mysql-backup.yaml b/k8s/dev/mysql-backup.yaml deleted file mode 100644 index b4859d90..00000000 --- a/k8s/dev/mysql-backup.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. -# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgresql-backup - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: postgresql-backup - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: postgresql-backup - template: - metadata: - labels: - app.kubernetes.io/name: postgresql-backup - app.kubernetes.io/part-of: udc-bot - environment: dev - spec: - containers: - - name: postgresql-backup - image: prodrigestivill/postgres-backup-local:16 - env: - - name: POSTGRES_HOST - value: postgresql - - name: POSTGRES_PORT - value: "5432" - - name: POSTGRES_DB - value: udcbot - - name: POSTGRES_USER - value: udcbot - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgresql-credentials - key: password - - name: SCHEDULE - value: "@daily" - - name: BACKUP_KEEP_DAYS - value: "90" - - name: BACKUP_DIR - value: /backups - volumeMounts: - - name: backup-data - mountPath: /backups - resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumes: - - name: backup-data - emptyDir: {} diff --git a/k8s/dev/mysql.yaml b/k8s/dev/mysql.yaml deleted file mode 100644 index 33a9eba9..00000000 --- a/k8s/dev/mysql.yaml +++ /dev/null @@ -1,110 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgresql-data - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgresql - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: postgresql - template: - metadata: - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: dev - spec: - containers: - - name: postgresql - image: postgres:16 - ports: - - containerPort: 5432 - protocol: TCP - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgresql-credentials - key: password - - name: POSTGRES_DB - value: udcbot - - name: POSTGRES_USER - value: udcbot - volumeMounts: - - name: postgresql-data - mountPath: /var/lib/postgresql/data - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - readinessProbe: - exec: - command: - - pg_isready - - -U - - udcbot - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - livenessProbe: - exec: - command: - - pg_isready - - -U - - udcbot - initialDelaySeconds: 30 - periodSeconds: 15 - timeoutSeconds: 5 - failureThreshold: 3 - securityContext: - allowPrivilegeEscalation: false - volumes: - - name: postgresql-data - persistentVolumeClaim: - claimName: postgresql-data ---- -apiVersion: v1 -kind: Service -metadata: - name: postgresql - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - type: ClusterIP - ports: - - port: 5432 - targetPort: 5432 - protocol: TCP - selector: - app.kubernetes.io/name: postgresql diff --git a/k8s/dev/pgadmin.yaml b/k8s/dev/pgadmin.yaml index f6c01f71..3da07699 100644 --- a/k8s/dev/pgadmin.yaml +++ b/k8s/dev/pgadmin.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: pgadmin - image: dpage/pgadmin4:8 + image: dpage/pgadmin4:8.14 ports: - containerPort: 80 protocol: TCP diff --git a/k8s/dev/phpmyadmin.yaml b/k8s/dev/phpmyadmin.yaml deleted file mode 100644 index f6c01f71..00000000 --- a/k8s/dev/phpmyadmin.yaml +++ /dev/null @@ -1,99 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgadmin - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: pgadmin - template: - metadata: - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev - spec: - containers: - - name: pgadmin - image: dpage/pgadmin4:8 - ports: - - containerPort: 80 - protocol: TCP - env: - - name: PGADMIN_DEFAULT_EMAIL - value: admin@udc.ovh - - name: PGADMIN_DEFAULT_PASSWORD - valueFrom: - secretKeyRef: - name: pgadmin-credentials - key: password - resources: - requests: - cpu: 25m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - readinessProbe: - httpGet: - path: /misc/ping - port: 80 - initialDelaySeconds: 10 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false ---- -apiVersion: v1 -kind: Service -metadata: - name: pgadmin - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: 80 - protocol: TCP - selector: - app.kubernetes.io/name: pgadmin ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: pgadmin - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd -spec: - ingressClassName: traefik - tls: - - hosts: - - pgadmin.dev.bot.udc.ovh - secretName: pgadmin-dev-tls - rules: - - host: pgadmin.dev.bot.udc.ovh - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: pgadmin - port: - number: 80 diff --git a/k8s/dev/postgresql-backup.yaml b/k8s/dev/postgresql-backup.yaml index b4859d90..cc56f546 100644 --- a/k8s/dev/postgresql-backup.yaml +++ b/k8s/dev/postgresql-backup.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: postgresql-backup - image: prodrigestivill/postgres-backup-local:16 + image: prodrigestivill/postgres-backup-local:16-alpine-d257e5d env: - name: POSTGRES_HOST value: postgresql diff --git a/k8s/dev/postgresql.yaml b/k8s/dev/postgresql.yaml index 33a9eba9..37f25272 100644 --- a/k8s/dev/postgresql.yaml +++ b/k8s/dev/postgresql.yaml @@ -40,11 +40,13 @@ spec: spec: containers: - name: postgresql - image: postgres:16 + image: postgres:16.13 ports: - containerPort: 5432 protocol: TCP env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: diff --git a/k8s/prod/mysql-backup.yaml b/k8s/prod/mysql-backup.yaml deleted file mode 100644 index 91f77069..00000000 --- a/k8s/prod/mysql-backup.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. -# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgresql-backup - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: postgresql-backup - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: postgresql-backup - template: - metadata: - labels: - app.kubernetes.io/name: postgresql-backup - app.kubernetes.io/part-of: udc-bot - environment: prod - spec: - containers: - - name: postgresql-backup - image: prodrigestivill/postgres-backup-local:16 - env: - - name: POSTGRES_HOST - value: postgresql - - name: POSTGRES_PORT - value: "5432" - - name: POSTGRES_DB - value: udcbot - - name: POSTGRES_USER - value: udcbot - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgresql-credentials - key: password - - name: SCHEDULE - value: "@daily" - - name: BACKUP_KEEP_DAYS - value: "90" - - name: BACKUP_DIR - value: /backups - volumeMounts: - - name: backup-data - mountPath: /backups - resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumes: - - name: backup-data - emptyDir: {} diff --git a/k8s/prod/mysql.yaml b/k8s/prod/mysql.yaml deleted file mode 100644 index 3bcd5dcd..00000000 --- a/k8s/prod/mysql.yaml +++ /dev/null @@ -1,110 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgresql-data - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgresql - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: postgresql - template: - metadata: - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: prod - spec: - containers: - - name: postgresql - image: postgres:16 - ports: - - containerPort: 5432 - protocol: TCP - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgresql-credentials - key: password - - name: POSTGRES_DB - value: udcbot - - name: POSTGRES_USER - value: udcbot - volumeMounts: - - name: postgresql-data - mountPath: /var/lib/postgresql/data - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 1000m - memory: 1Gi - readinessProbe: - exec: - command: - - pg_isready - - -U - - udcbot - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - livenessProbe: - exec: - command: - - pg_isready - - -U - - udcbot - initialDelaySeconds: 30 - periodSeconds: 15 - timeoutSeconds: 5 - failureThreshold: 3 - securityContext: - allowPrivilegeEscalation: false - volumes: - - name: postgresql-data - persistentVolumeClaim: - claimName: postgresql-data ---- -apiVersion: v1 -kind: Service -metadata: - name: postgresql - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: postgresql - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - type: ClusterIP - ports: - - port: 5432 - targetPort: 5432 - protocol: TCP - selector: - app.kubernetes.io/name: postgresql diff --git a/k8s/prod/pgadmin.yaml b/k8s/prod/pgadmin.yaml index ac8b04cb..256f2837 100644 --- a/k8s/prod/pgadmin.yaml +++ b/k8s/prod/pgadmin.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: pgadmin - image: dpage/pgadmin4:8 + image: dpage/pgadmin4:8.14 ports: - containerPort: 80 protocol: TCP diff --git a/k8s/prod/phpmyadmin.yaml b/k8s/prod/phpmyadmin.yaml deleted file mode 100644 index ac8b04cb..00000000 --- a/k8s/prod/phpmyadmin.yaml +++ /dev/null @@ -1,99 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgadmin - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: pgadmin - template: - metadata: - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod - spec: - containers: - - name: pgadmin - image: dpage/pgadmin4:8 - ports: - - containerPort: 80 - protocol: TCP - env: - - name: PGADMIN_DEFAULT_EMAIL - value: admin@udc.ovh - - name: PGADMIN_DEFAULT_PASSWORD - valueFrom: - secretKeyRef: - name: pgadmin-credentials - key: password - resources: - requests: - cpu: 25m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - readinessProbe: - httpGet: - path: /misc/ping - port: 80 - initialDelaySeconds: 10 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false ---- -apiVersion: v1 -kind: Service -metadata: - name: pgadmin - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: 80 - protocol: TCP - selector: - app.kubernetes.io/name: pgadmin ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: pgadmin - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd -spec: - ingressClassName: traefik - tls: - - hosts: - - pgadmin.bot.udc.ovh - secretName: pgadmin-prod-tls - rules: - - host: pgadmin.bot.udc.ovh - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: pgadmin - port: - number: 80 diff --git a/k8s/prod/postgresql-backup.yaml b/k8s/prod/postgresql-backup.yaml index 91f77069..255344f4 100644 --- a/k8s/prod/postgresql-backup.yaml +++ b/k8s/prod/postgresql-backup.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: postgresql-backup - image: prodrigestivill/postgres-backup-local:16 + image: prodrigestivill/postgres-backup-local:16-alpine-d257e5d env: - name: POSTGRES_HOST value: postgresql diff --git a/k8s/prod/postgresql.yaml b/k8s/prod/postgresql.yaml index 3bcd5dcd..782519ac 100644 --- a/k8s/prod/postgresql.yaml +++ b/k8s/prod/postgresql.yaml @@ -40,11 +40,13 @@ spec: spec: containers: - name: postgresql - image: postgres:16 + image: postgres:16.13 ports: - containerPort: 5432 protocol: TCP env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: From 8b80c668b151f9b12cf15249ecace2bbf10653f2 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Wed, 1 Apr 2026 20:22:39 +0200 Subject: [PATCH 03/25] feat(k8s): add AWS credentials for PostgreSQL backups --- k8s/dev/external-secrets.yaml | 23 ++++++++++++++++++++ k8s/dev/postgresql-backup.yaml | 38 +++++++++++++++++++++------------ k8s/prod/external-secrets.yaml | 23 ++++++++++++++++++++ k8s/prod/postgresql-backup.yaml | 38 +++++++++++++++++++++------------ 4 files changed, 94 insertions(+), 28 deletions(-) diff --git a/k8s/dev/external-secrets.yaml b/k8s/dev/external-secrets.yaml index 5dbde529..7f94fa6b 100644 --- a/k8s/dev/external-secrets.yaml +++ b/k8s/dev/external-secrets.yaml @@ -90,3 +90,26 @@ spec: remoteRef: key: "AirlabAPI" property: api_key +--- +# AWS credentials for S3 backups — from 1Password "AWS Backup Credentials" +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgresql-backup-credentials + namespace: udc-bot-dev +spec: + refreshInterval: 1h + secretStoreRef: + name: onepassword + kind: ClusterSecretStore + target: + name: postgresql-backup-credentials + data: + - secretKey: AWS_ACCESS_KEY_ID + remoteRef: + key: "AWS Backup Credentials" + property: access-key-id + - secretKey: AWS_SECRET_ACCESS_KEY + remoteRef: + key: "AWS Backup Credentials" + property: secret-access-key diff --git a/k8s/dev/postgresql-backup.yaml b/k8s/dev/postgresql-backup.yaml index cc56f546..4c7936c0 100644 --- a/k8s/dev/postgresql-backup.yaml +++ b/k8s/dev/postgresql-backup.yaml @@ -1,5 +1,5 @@ -# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. -# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. +# PostgreSQL backup using eeshugerman/postgres-backup-s3 — pg_dump directly to S3. +# Dumps go to S3 bucket "udc-bot-postgresql-backups/dev" with 90-day retention. apiVersion: apps/v1 kind: Deployment metadata: @@ -23,15 +23,17 @@ spec: app.kubernetes.io/part-of: udc-bot environment: dev spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 containers: - name: postgresql-backup - image: prodrigestivill/postgres-backup-local:16-alpine-d257e5d + image: eeshugerman/postgres-backup-s3:16@sha256:59e6cef8459fafc73a5383e103309f23be84bd0bceb89c08b49ffc43780efd01 env: - name: POSTGRES_HOST value: postgresql - - name: POSTGRES_PORT - value: "5432" - - name: POSTGRES_DB + - name: POSTGRES_DATABASE value: udcbot - name: POSTGRES_USER value: udcbot @@ -40,15 +42,26 @@ spec: secretKeyRef: name: postgresql-credentials key: password + - name: S3_REGION + value: eu-west-3 + - name: S3_BUCKET + value: udc-bot-postgresql-backups + - name: S3_PREFIX + value: dev + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: postgresql-backup-credentials + key: AWS_ACCESS_KEY_ID + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: postgresql-backup-credentials + key: AWS_SECRET_ACCESS_KEY - name: SCHEDULE value: "@daily" - name: BACKUP_KEEP_DAYS value: "90" - - name: BACKUP_DIR - value: /backups - volumeMounts: - - name: backup-data - mountPath: /backups resources: requests: cpu: 25m @@ -61,6 +74,3 @@ spec: capabilities: drop: - ALL - volumes: - - name: backup-data - emptyDir: {} diff --git a/k8s/prod/external-secrets.yaml b/k8s/prod/external-secrets.yaml index f4db19db..ec546195 100644 --- a/k8s/prod/external-secrets.yaml +++ b/k8s/prod/external-secrets.yaml @@ -90,3 +90,26 @@ spec: remoteRef: key: "AirlabAPI" property: api_key +--- +# AWS credentials for S3 backups — from 1Password "AWS Backup Credentials" +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgresql-backup-credentials + namespace: udc-bot-prod +spec: + refreshInterval: 1h + secretStoreRef: + name: onepassword + kind: ClusterSecretStore + target: + name: postgresql-backup-credentials + data: + - secretKey: AWS_ACCESS_KEY_ID + remoteRef: + key: "AWS Backup Credentials" + property: access-key-id + - secretKey: AWS_SECRET_ACCESS_KEY + remoteRef: + key: "AWS Backup Credentials" + property: secret-access-key diff --git a/k8s/prod/postgresql-backup.yaml b/k8s/prod/postgresql-backup.yaml index 255344f4..876038eb 100644 --- a/k8s/prod/postgresql-backup.yaml +++ b/k8s/prod/postgresql-backup.yaml @@ -1,5 +1,5 @@ -# PostgreSQL backup using prodrigestivill/postgres-backup-local — handles daily schedule internally. -# Dumps go to S3 bucket "udc-bot-postgresql-backups" with 90-day retention via S3 lifecycle. +# PostgreSQL backup using eeshugerman/postgres-backup-s3 — pg_dump directly to S3. +# Dumps go to S3 bucket "udc-bot-postgresql-backups/prod" with 90-day retention. apiVersion: apps/v1 kind: Deployment metadata: @@ -23,15 +23,17 @@ spec: app.kubernetes.io/part-of: udc-bot environment: prod spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 containers: - name: postgresql-backup - image: prodrigestivill/postgres-backup-local:16-alpine-d257e5d + image: eeshugerman/postgres-backup-s3:16@sha256:59e6cef8459fafc73a5383e103309f23be84bd0bceb89c08b49ffc43780efd01 env: - name: POSTGRES_HOST value: postgresql - - name: POSTGRES_PORT - value: "5432" - - name: POSTGRES_DB + - name: POSTGRES_DATABASE value: udcbot - name: POSTGRES_USER value: udcbot @@ -40,15 +42,26 @@ spec: secretKeyRef: name: postgresql-credentials key: password + - name: S3_REGION + value: eu-west-3 + - name: S3_BUCKET + value: udc-bot-postgresql-backups + - name: S3_PREFIX + value: prod + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: postgresql-backup-credentials + key: AWS_ACCESS_KEY_ID + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: postgresql-backup-credentials + key: AWS_SECRET_ACCESS_KEY - name: SCHEDULE value: "@daily" - name: BACKUP_KEEP_DAYS value: "90" - - name: BACKUP_DIR - value: /backups - volumeMounts: - - name: backup-data - mountPath: /backups resources: requests: cpu: 25m @@ -61,6 +74,3 @@ spec: capabilities: drop: - ALL - volumes: - - name: backup-data - emptyDir: {} From 805bc648d1c3a0283ed78576bd331e1e741e748f Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 03:53:04 +0200 Subject: [PATCH 04/25] Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05e18277..994c7fc2 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,7 @@ sudo apt install ttf-mscorefonts-installer **Connection String Format:** ```json -"DbConnectionString": "Server=localhost;Database=your_db_name;Uid=your_username;Pwd=your_password;" +"DbConnectionString": "Host=localhost;Port=5432;Database=your_db_name;Username=your_username;Password=your_password" ``` ## Notes From 9c45b010a43e8f24ba2460bb3ea7e64bb8449874 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 17:58:09 +0200 Subject: [PATCH 05/25] docs(README): update database container command in instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 994c7fc2..a8bc7732 100644 --- a/README.md +++ b/README.md @@ -197,12 +197,12 @@ _For production deployment, consult the [Discord.Net Deployment Guide](https://d docker-compose up # Start only the database (run bot from IDE for faster development) -docker-compose up database +docker-compose up db ``` **Development workflow:** -1. Start the database container: `docker-compose up database` +1. Start the database container: `docker-compose up db` 2. Update the `DbConnectionString` in `Settings.json` to match your docker-compose configuration 3. Run the bot from your IDE for faster development iteration From 44caba09d89b33de97db681bf1a8bee30ada97cc Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 17:59:01 +0200 Subject: [PATCH 06/25] refactor(database): simplify SQL queries in services --- .../Extensions/DBConnectionExtension.cs | 5 +- DiscordBot/Services/DatabaseService.cs | 56 +++++++++---------- DiscordBot/Services/KarmaResetService.cs | 18 +++--- 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/DiscordBot/Extensions/DBConnectionExtension.cs b/DiscordBot/Extensions/DBConnectionExtension.cs index 44acc354..9294ebd0 100644 --- a/DiscordBot/Extensions/DBConnectionExtension.cs +++ b/DiscordBot/Extensions/DBConnectionExtension.cs @@ -1,5 +1,6 @@ using System.Data.Common; using Insight.Database; +using Npgsql; namespace DiscordBot.Extensions; @@ -7,8 +8,8 @@ public static class DBConnectionExtension { public static async Task ColumnExists(this DbConnection connection, string tableName, string columnName) { - var query = $"SELECT 1 FROM information_schema.columns WHERE table_name = '{tableName}' AND column_name = '{columnName}'"; - var response = await connection.QuerySqlAsync(query); + const string query = "SELECT 1 FROM information_schema.columns WHERE LOWER(table_name) = LOWER(@tableName) AND LOWER(column_name) = LOWER(@columnName)"; + var response = await connection.QuerySqlAsync(query, new { tableName, columnName }); return response.Count > 0; } } \ No newline at end of file diff --git a/DiscordBot/Services/DatabaseService.cs b/DiscordBot/Services/DatabaseService.cs index 11549fe9..f073ddf0 100644 --- a/DiscordBot/Services/DatabaseService.cs +++ b/DiscordBot/Services/DatabaseService.cs @@ -78,7 +78,7 @@ await _logging.LogAction( var defaultCityExists = await c.ColumnExists(UserProps.TableName, UserProps.DefaultCity); if (!defaultCityExists) { - c.ExecuteSql($"ALTER TABLE \"{UserProps.TableName}\" ADD COLUMN \"{UserProps.DefaultCity}\" varchar(64) DEFAULT NULL"); + c.ExecuteSql($"ALTER TABLE {UserProps.TableName} ADD COLUMN {UserProps.DefaultCity} varchar(64) DEFAULT NULL"); await _logging.LogAction($"DatabaseService: Added missing column '{UserProps.DefaultCity}' to table '{UserProps.TableName}'.", ExtendedLogSeverity.Positive); } @@ -90,17 +90,17 @@ await _logging.LogAction($"DatabaseService: Table '{UserProps.TableName}' does n try { c.ExecuteSql( - $"CREATE TABLE \"{UserProps.TableName}\" (" + - $"\"ID\" SERIAL PRIMARY KEY, " + - $"\"{UserProps.UserID}\" varchar(32) NOT NULL UNIQUE, " + - $"\"{UserProps.Karma}\" integer NOT NULL DEFAULT 0, " + - $"\"{UserProps.KarmaWeekly}\" integer NOT NULL DEFAULT 0, " + - $"\"{UserProps.KarmaMonthly}\" integer NOT NULL DEFAULT 0, " + - $"\"{UserProps.KarmaYearly}\" integer NOT NULL DEFAULT 0, " + - $"\"{UserProps.KarmaGiven}\" integer NOT NULL DEFAULT 0, " + - $"\"{UserProps.Exp}\" bigint NOT NULL DEFAULT 0, " + - $"\"{UserProps.Level}\" integer NOT NULL DEFAULT 0, " + - $"\"{UserProps.DefaultCity}\" varchar(64) DEFAULT NULL)"); + $"CREATE TABLE {UserProps.TableName} (" + + $"id SERIAL PRIMARY KEY, " + + $"{UserProps.UserID} varchar(32) NOT NULL UNIQUE, " + + $"{UserProps.Karma} integer NOT NULL DEFAULT 0, " + + $"{UserProps.KarmaWeekly} integer NOT NULL DEFAULT 0, " + + $"{UserProps.KarmaMonthly} integer NOT NULL DEFAULT 0, " + + $"{UserProps.KarmaYearly} integer NOT NULL DEFAULT 0, " + + $"{UserProps.KarmaGiven} integer NOT NULL DEFAULT 0, " + + $"{UserProps.Exp} bigint NOT NULL DEFAULT 0, " + + $"{UserProps.Level} integer NOT NULL DEFAULT 0, " + + $"{UserProps.DefaultCity} varchar(64) DEFAULT NULL)"); } catch (Exception e) { @@ -130,26 +130,26 @@ await _logging.LogAction( try { c.ExecuteSql( - $"CREATE TABLE \"{CasinoProps.CasinoTableName}\" (" + - $"\"{CasinoProps.Id}\" SERIAL PRIMARY KEY, " + - $"\"{CasinoProps.UserID}\" varchar(32) NOT NULL UNIQUE, " + - $"\"{CasinoProps.Tokens}\" bigint NOT NULL DEFAULT 1000, " + - $"\"{CasinoProps.CreatedAt}\" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, " + - $"\"{CasinoProps.UpdatedAt}\" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, " + - $"\"{CasinoProps.LastDailyReward}\" timestamptz NOT NULL DEFAULT '1970-01-01 00:00:01+00')"); + $"CREATE TABLE {CasinoProps.CasinoTableName} (" + + $"{CasinoProps.Id} SERIAL PRIMARY KEY, " + + $"{CasinoProps.UserID} varchar(32) NOT NULL UNIQUE, " + + $"{CasinoProps.Tokens} bigint NOT NULL DEFAULT 1000, " + + $"{CasinoProps.CreatedAt} timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, " + + $"{CasinoProps.UpdatedAt} timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, " + + $"{CasinoProps.LastDailyReward} timestamptz NOT NULL DEFAULT '1970-01-01 00:00:01+00')"); c.ExecuteSql( - $"CREATE TABLE \"{CasinoProps.TransactionTableName}\" (" + - $"\"{CasinoProps.TransactionId}\" SERIAL PRIMARY KEY, " + - $"\"{CasinoProps.TransactionUserID}\" varchar(32) NOT NULL, " + - $"\"{CasinoProps.Amount}\" bigint NOT NULL, " + - $"\"{CasinoProps.TransactionType}\" integer NOT NULL, " + - $"\"{CasinoProps.Details}\" jsonb DEFAULT NULL, " + - $"\"{CasinoProps.TransactionCreatedAt}\" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)"); + $"CREATE TABLE {CasinoProps.TransactionTableName} (" + + $"{CasinoProps.TransactionId} SERIAL PRIMARY KEY, " + + $"{CasinoProps.TransactionUserID} varchar(32) NOT NULL, " + + $"{CasinoProps.Amount} bigint NOT NULL, " + + $"{CasinoProps.TransactionType} integer NOT NULL, " + + $"{CasinoProps.Details} jsonb DEFAULT NULL, " + + $"{CasinoProps.TransactionCreatedAt} timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)"); c.ExecuteSql( - $"CREATE INDEX idx_user_created ON \"{CasinoProps.TransactionTableName}\" " + - $"(\"{CasinoProps.TransactionUserID}\", \"{CasinoProps.TransactionCreatedAt}\")"); + $"CREATE INDEX idx_user_created ON {CasinoProps.TransactionTableName} " + + $"({CasinoProps.TransactionUserID}, {CasinoProps.TransactionCreatedAt})"); } catch (Exception e) { diff --git a/DiscordBot/Services/KarmaResetService.cs b/DiscordBot/Services/KarmaResetService.cs index d47f82e7..69f8d3ea 100644 --- a/DiscordBot/Services/KarmaResetService.cs +++ b/DiscordBot/Services/KarmaResetService.cs @@ -69,12 +69,12 @@ private async Task EnsureMetaTable() await using var c = new NpgsqlConnection(_connectionString); await c.OpenAsync(); await c.ExecuteSqlAsync( - $"CREATE TABLE IF NOT EXISTS \"{MetaTable}\" (" + - $"\"period\" varchar(16) PRIMARY KEY, " + - $"\"last_reset\" timestamptz NOT NULL DEFAULT '1970-01-01 00:00:00+00')"); - await c.ExecuteSqlAsync($"INSERT INTO \"{MetaTable}\" (\"period\") VALUES ('weekly') ON CONFLICT DO NOTHING"); - await c.ExecuteSqlAsync($"INSERT INTO \"{MetaTable}\" (\"period\") VALUES ('monthly') ON CONFLICT DO NOTHING"); - await c.ExecuteSqlAsync($"INSERT INTO \"{MetaTable}\" (\"period\") VALUES ('yearly') ON CONFLICT DO NOTHING"); + $"CREATE TABLE IF NOT EXISTS {MetaTable} (" + + $"period varchar(16) PRIMARY KEY, " + + $"last_reset timestamptz NOT NULL DEFAULT '1970-01-01 00:00:00+00')"); + await c.ExecuteSqlAsync($"INSERT INTO {MetaTable} (period) VALUES ('weekly') ON CONFLICT DO NOTHING"); + await c.ExecuteSqlAsync($"INSERT INTO {MetaTable} (period) VALUES ('monthly') ON CONFLICT DO NOTHING"); + await c.ExecuteSqlAsync($"INSERT INTO {MetaTable} (period) VALUES ('yearly') ON CONFLICT DO NOTHING"); } private async Task CatchUpMissedResets() @@ -115,8 +115,8 @@ private async Task ResetColumn(string period, string column) { await using var c = new NpgsqlConnection(_connectionString); await c.OpenAsync(); - await c.ExecuteSqlAsync($"UPDATE \"{UserProps.TableName}\" SET \"{column}\" = 0"); - await c.ExecuteSqlAsync($"UPDATE \"{MetaTable}\" SET \"last_reset\" = NOW() WHERE \"period\" = '{period}'"); + await c.ExecuteSqlAsync($"UPDATE {UserProps.TableName} SET {column} = 0"); + await c.ExecuteSqlAsync($"UPDATE {MetaTable} SET last_reset = NOW() WHERE period = @period", new { period }); await _logging.LogChannelAndFile($"KarmaResetService: Reset {period} karma ({column}).", ExtendedLogSeverity.Positive); } @@ -124,7 +124,7 @@ private async Task GetLastReset(string period) { await using var c = new NpgsqlConnection(_connectionString); await c.OpenAsync(); - var results = await c.QuerySqlAsync($"SELECT \"last_reset\" FROM \"{MetaTable}\" WHERE \"period\" = '{period}'"); + var results = await c.QuerySqlAsync($"SELECT last_reset FROM {MetaTable} WHERE period = @period", new { period }); if (results.Count > 0 && results[0] is IDictionary row && row.TryGetValue("last_reset", out var val) && val is DateTime dt) return dt; return DateTime.MinValue; From 1811d8d1ce1a617a171b70bdc960093fc4a56cc2 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 17:59:07 +0200 Subject: [PATCH 07/25] chore(deps): update Insight.Database to version 8.0.6 --- DiscordBot/DiscordBot.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DiscordBot/DiscordBot.csproj b/DiscordBot/DiscordBot.csproj index 97731195..8d7e9c26 100644 --- a/DiscordBot/DiscordBot.csproj +++ b/DiscordBot/DiscordBot.csproj @@ -6,7 +6,7 @@ enable - + From 58779533f40bdcd809391f234df38e1d28d02540 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 19:27:52 +0200 Subject: [PATCH 08/25] chore(k8s): update bot image to specific version 832a135 --- k8s/dev/bot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/dev/bot.yaml b/k8s/dev/bot.yaml index 11dc94be..d848126f 100644 --- a/k8s/dev/bot.yaml +++ b/k8s/dev/bot.yaml @@ -128,7 +128,7 @@ spec: - ALL containers: - name: bot - image: ghcr.io/unity-developer-community/udc-bot-dev:latest + image: ghcr.io/unity-developer-community/udc-bot-dev:832a135 volumeMounts: - name: app-settings mountPath: /app/Settings From 6dbfa7873b4d4b69a3f5050ee0f8db7d398ce921 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 20:46:24 +0200 Subject: [PATCH 09/25] chore(k8s): remove unnecessary middleware annotation from Ingress --- k8s/dev/pgadmin.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/dev/pgadmin.yaml b/k8s/dev/pgadmin.yaml index 3da07699..93278190 100644 --- a/k8s/dev/pgadmin.yaml +++ b/k8s/dev/pgadmin.yaml @@ -79,7 +79,6 @@ metadata: environment: dev annotations: cert-manager.io/cluster-issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd spec: ingressClassName: traefik tls: From bec3057fa04c0f95d97184b00327f49c8dd2ceef Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 21:03:01 +0200 Subject: [PATCH 10/25] feat(pgadmin): add configuration for PostgreSQL connection Co-authored-by: Copilot --- k8s/dev/pgadmin.yaml | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/k8s/dev/pgadmin.yaml b/k8s/dev/pgadmin.yaml index 93278190..3845bbf8 100644 --- a/k8s/dev/pgadmin.yaml +++ b/k8s/dev/pgadmin.yaml @@ -1,4 +1,30 @@ --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgadmin-servers + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: dev +data: + servers.json: | + { + "Servers": { + "1": { + "Name": "UDC Bot - PostgreSQL (Dev)", + "Group": "Servers", + "Host": "postgresql", + "Port": 5432, + "MaintenanceDB": "udcbot", + "Username": "udcbot", + "SSLMode": "prefer", + "PassFile": "/tmp/pgpass" + } + } + } +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -20,6 +46,26 @@ spec: app.kubernetes.io/part-of: udc-bot environment: dev spec: + initContainers: + - name: create-pgpass + image: busybox:1.36 + command: + - sh + - -c + - | + echo "postgresql:5432:udcbot:udcbot:${PGPASSWORD}" > /pgpass/pgpass + chmod 600 /pgpass/pgpass + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgresql-credentials + key: password + volumeMounts: + - name: pgpass + mountPath: /pgpass + securityContext: + allowPrivilegeEscalation: false containers: - name: pgadmin image: dpage/pgadmin4:8.14 @@ -34,6 +80,17 @@ spec: secretKeyRef: name: pgadmin-credentials key: password + - name: PGADMIN_SERVER_JSON_FILE + value: /pgadmin4/servers.json + volumeMounts: + - name: servers-config + mountPath: /pgadmin4/servers.json + subPath: servers.json + readOnly: true + - name: pgpass + mountPath: /tmp/pgpass + subPath: pgpass + readOnly: true resources: requests: cpu: 25m @@ -49,6 +106,12 @@ spec: periodSeconds: 10 securityContext: allowPrivilegeEscalation: false + volumes: + - name: servers-config + configMap: + name: pgadmin-servers + - name: pgpass + emptyDir: {} --- apiVersion: v1 kind: Service From 20cfffae71622c528b6ce188ee0e6397f3bfa693 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 21:56:20 +0200 Subject: [PATCH 11/25] feat(pgadmin): add PostgreSQL server configuration and init container for prod --- k8s/prod/pgadmin.yaml | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/k8s/prod/pgadmin.yaml b/k8s/prod/pgadmin.yaml index 256f2837..edac4f38 100644 --- a/k8s/prod/pgadmin.yaml +++ b/k8s/prod/pgadmin.yaml @@ -1,4 +1,30 @@ --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgadmin-servers + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: pgadmin + app.kubernetes.io/part-of: udc-bot + environment: prod +data: + servers.json: | + { + "Servers": { + "1": { + "Name": "UDC Bot - PostgreSQL (Prod)", + "Group": "Servers", + "Host": "postgresql", + "Port": 5432, + "MaintenanceDB": "udcbot", + "Username": "udcbot", + "SSLMode": "prefer", + "PassFile": "/tmp/pgpass" + } + } + } +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -20,6 +46,26 @@ spec: app.kubernetes.io/part-of: udc-bot environment: prod spec: + initContainers: + - name: create-pgpass + image: busybox:1.36 + command: + - sh + - -c + - | + echo "postgresql:5432:udcbot:udcbot:${PGPASSWORD}" > /pgpass/pgpass + chmod 600 /pgpass/pgpass + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgresql-credentials + key: password + volumeMounts: + - name: pgpass + mountPath: /pgpass + securityContext: + allowPrivilegeEscalation: false containers: - name: pgadmin image: dpage/pgadmin4:8.14 @@ -34,6 +80,17 @@ spec: secretKeyRef: name: pgadmin-credentials key: password + - name: PGADMIN_SERVER_JSON_FILE + value: /pgadmin4/servers.json + volumeMounts: + - name: servers-config + mountPath: /pgadmin4/servers.json + subPath: servers.json + readOnly: true + - name: pgpass + mountPath: /tmp/pgpass + subPath: pgpass + readOnly: true resources: requests: cpu: 25m @@ -49,6 +106,12 @@ spec: periodSeconds: 10 securityContext: allowPrivilegeEscalation: false + volumes: + - name: servers-config + configMap: + name: pgadmin-servers + - name: pgpass + emptyDir: {} --- apiVersion: v1 kind: Service From f0c6e411406475b7d52f654634a629c5713c5290 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 21:56:33 +0200 Subject: [PATCH 12/25] feat(pgadmin): add pgAdmin server configuration and credentials for dockercompose --- .gitignore | 3 +++ docker-compose.yml | 10 ++++++++++ pgadmin/servers.json | 14 ++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 pgadmin/servers.json diff --git a/.gitignore b/.gitignore index 9908e11e..0fd898e5 100644 --- a/.gitignore +++ b/.gitignore @@ -455,3 +455,6 @@ fabric.properties # Bot runtime-generated data (SERVER/ is created by the bot at runtime) DiscordBot/SERVER/ + +# pgAdmin pgpass contains plaintext credentials +pgadmin/pgpass diff --git a/docker-compose.yml b/docker-compose.yml index 77740b80..c9c291d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,16 @@ services: environment: PGADMIN_DEFAULT_EMAIL: admin@local.dev PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_SERVER_JSON_FILE: /pgadmin4/servers.json + volumes: + - ./pgadmin/servers.json:/pgadmin4/servers.json:ro + - ./pgadmin/pgpass:/pgpass_src:ro + entrypoint: > + /bin/sh -c " + cp /pgpass_src /tmp/pgpass && + chmod 600 /tmp/pgpass && + /entrypoint.sh + " bot: build: . diff --git a/pgadmin/servers.json b/pgadmin/servers.json new file mode 100644 index 00000000..2bcccea1 --- /dev/null +++ b/pgadmin/servers.json @@ -0,0 +1,14 @@ +{ + "Servers": { + "1": { + "Name": "UDC Bot - PostgreSQL", + "Group": "Servers", + "Host": "db", + "Port": 5432, + "MaintenanceDB": "udcbot", + "Username": "udcbot", + "SSLMode": "prefer", + "PassFile": "/tmp/pgpass" + } + } +} From 17fc1564a5b5c0df39e1ecdcd21a3339e852f111 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 21:57:00 +0200 Subject: [PATCH 13/25] refactor(casino): change types from ulong to long since postgre doesn't support ulong --- DiscordBot/Domain/Casino/CasinoUser.cs | 2 +- DiscordBot/Domain/Casino/Game.cs | 6 ++-- DiscordBot/Domain/Casino/GamePlayer.cs | 2 +- DiscordBot/Domain/Casino/GameSession.cs | 10 +++--- .../Casino/Games/Cards/Blackjack/Blackjack.cs | 6 ++-- .../Domain/Casino/Games/Cards/Poker/Poker.cs | 11 +++--- .../RockPaperScissors/RockPaperScissors.cs | 8 ++--- DiscordBot/Domain/ProfileData.cs | 14 ++++---- DiscordBot/Extensions/CasinoRepository.cs | 4 +-- DiscordBot/Extensions/UserDBRepository.cs | 34 +++++++++---------- .../Modules/Casino/CasinoSlashModule.Games.cs | 4 +-- .../Modules/Casino/CasinoSlashModule.cs | 8 ++--- DiscordBot/Modules/UserModule.cs | 4 +-- DiscordBot/Services/Casino/CasinoService.cs | 20 +++++------ DiscordBot/Services/Casino/GameService.cs | 2 +- DiscordBot/Services/UserService.cs | 18 +++++----- DiscordBot/Settings/Deserialized/Settings.cs | 4 +-- 17 files changed, 78 insertions(+), 79 deletions(-) diff --git a/DiscordBot/Domain/Casino/CasinoUser.cs b/DiscordBot/Domain/Casino/CasinoUser.cs index d4aa664e..9f7e61ab 100644 --- a/DiscordBot/Domain/Casino/CasinoUser.cs +++ b/DiscordBot/Domain/Casino/CasinoUser.cs @@ -6,7 +6,7 @@ public class CasinoUser { public int Id { get; set; } public required string UserID { get; set; } - public ulong Tokens { get; set; } + public long Tokens { get; set; } public DateTime CreatedAt { get; set; } public DateTime UpdatedAt { get; set; } public DateTime LastDailyReward { get; set; } diff --git a/DiscordBot/Domain/Casino/Game.cs b/DiscordBot/Domain/Casino/Game.cs index c50fafe0..85478fad 100644 --- a/DiscordBot/Domain/Casino/Game.cs +++ b/DiscordBot/Domain/Casino/Game.cs @@ -56,7 +56,7 @@ public interface ICasinoGame public IReadOnlyList<(GamePlayer player, long payout)> EndGame(); public abstract GamePlayerResult GetPlayerGameResult(GamePlayer player); - public abstract long CalculatePayout(GamePlayer player, ulong totalPot); + public abstract long CalculatePayout(GamePlayer player, long totalPot); public void Reset(); @@ -149,8 +149,8 @@ public void StartGame(IEnumerable players) // Default implementation does nothing, override in specific games if needed protected virtual void FinalizeGame(List players) { } public abstract GamePlayerResult GetPlayerGameResult(GamePlayer player); - protected ulong GetTotalPot => (ulong)Players.Sum(p => (long)p.Bet); - public abstract long CalculatePayout(GamePlayer player, ulong totalPot); + protected long GetTotalPot => Players.Sum(p => p.Bet); + public abstract long CalculatePayout(GamePlayer player, long totalPot); /// /// Determines if the game should enter the FINISHED state.
diff --git a/DiscordBot/Domain/Casino/GamePlayer.cs b/DiscordBot/Domain/Casino/GamePlayer.cs index fab05a10..3ecb33e6 100644 --- a/DiscordBot/Domain/Casino/GamePlayer.cs +++ b/DiscordBot/Domain/Casino/GamePlayer.cs @@ -16,7 +16,7 @@ public class GamePlayer /// /// The bet amount placed by the player /// - public required ulong Bet { get; set; } + public required long Bet { get; set; } /// /// The final result of the player in the game (won, lost, tie) /// diff --git a/DiscordBot/Domain/Casino/GameSession.cs b/DiscordBot/Domain/Casino/GameSession.cs index 3294d156..6fb29dc9 100644 --- a/DiscordBot/Domain/Casino/GameSession.cs +++ b/DiscordBot/Domain/Casino/GameSession.cs @@ -10,12 +10,12 @@ public interface IGameSession public Type ActionType { get; } public DiscordGamePlayer? GetPlayer(ulong userId); - public bool AddPlayer(ulong userId, ulong bet); + public bool AddPlayer(ulong userId, long bet); public bool AddPlayerAI(); public void RemovePlayer(ulong userId); public void RemovePlayerAI(); public void SetPlayerReady(ulong userId, bool isReady); - public void SetPlayerBet(ulong userId, ulong bet); + public void SetPlayerBet(ulong userId, long bet); public void DoPlayerAction(ulong userId, Enum action); public bool HasNextDealerAction(); @@ -75,7 +75,7 @@ public GameSession(TGame game, int maxSeats) /// public bool CanStart => Game.State == GameState.NotStarted && PlayerCount >= Game.MinPlayers && AllPlayersReady; - public ulong GetTotalPot => (ulong)Players.Sum(p => (long)p.Bet); + public long GetTotalPot => Players.Sum(p => p.Bet); public bool ShouldFinish() => Game.ShouldFinish(); @@ -94,7 +94,7 @@ public void Reset() public DiscordGamePlayer? GetPlayer(ulong userId) => Players.FirstOrDefault(p => p.UserId == userId); - public bool AddPlayer(ulong userId, ulong bet) + public bool AddPlayer(ulong userId, long bet) { if (!CanJoin) return false; if (Players.Any(p => p.UserId == userId)) return false; // Player already in game @@ -156,7 +156,7 @@ public void SetPlayerReady(ulong userId, bool ready = true) if (ready && CanStart) Game.StartGame(Players); } - public void SetPlayerBet(ulong userId, ulong bet) + public void SetPlayerBet(ulong userId, long bet) { if (Game.State != GameState.NotStarted) return; // Cannot change bet after the game has started var player = GetPlayer(userId); diff --git a/DiscordBot/Domain/Casino/Games/Cards/Blackjack/Blackjack.cs b/DiscordBot/Domain/Casino/Games/Cards/Blackjack/Blackjack.cs index 06f4e70a..15d2682b 100644 --- a/DiscordBot/Domain/Casino/Games/Cards/Blackjack/Blackjack.cs +++ b/DiscordBot/Domain/Casino/Games/Cards/Blackjack/Blackjack.cs @@ -105,12 +105,12 @@ public override GamePlayerResult GetPlayerGameResult(GamePlayer player) return GamePlayerResult.NoResult; } - public override long CalculatePayout(GamePlayer player, ulong _totalPot) + public override long CalculatePayout(GamePlayer player, long _totalPot) { return player.Result switch { - GamePlayerResult.Won => (long)player.Bet, - GamePlayerResult.Lost => -(long)player.Bet, + GamePlayerResult.Won => player.Bet, + GamePlayerResult.Lost => -player.Bet, GamePlayerResult.Tie => 0, _ => 0 }; diff --git a/DiscordBot/Domain/Casino/Games/Cards/Poker/Poker.cs b/DiscordBot/Domain/Casino/Games/Cards/Poker/Poker.cs index 3f28f866..3a4a71e3 100644 --- a/DiscordBot/Domain/Casino/Games/Cards/Poker/Poker.cs +++ b/DiscordBot/Domain/Casino/Games/Cards/Poker/Poker.cs @@ -230,11 +230,11 @@ public override GamePlayerResult GetPlayerGameResult(GamePlayer player) return winners.Any(w => w.player == player) ? GamePlayerResult.Won : GamePlayerResult.Lost; } - public override long CalculatePayout(GamePlayer player, ulong totalPot) + public override long CalculatePayout(GamePlayer player, long totalPot) { var result = GetPlayerGameResult(player); if (result == GamePlayerResult.Lost) - return -(long)player.Bet; + return -player.Bet; // Calculate winner's share var allHands = Players.Where(p => GameData[p].FinalHand != null) @@ -244,12 +244,11 @@ public override long CalculatePayout(GamePlayer player, ulong totalPot) if (winner.player != null) { - // Winner gets their share of the total pot minus their original bet - var winnings = (long)(totalPot * (ulong)winner.share); - return winnings - (long)player.Bet; + var winnings = (long)(totalPot * winner.share); + return winnings - player.Bet; } - return -(long)player.Bet; + return -player.Bet; } public override bool ShouldFinish() => State == GameState.InProgress && Players.All(p => GameData[p].HasDiscarded); diff --git a/DiscordBot/Domain/Casino/Games/RockPaperScissors/RockPaperScissors.cs b/DiscordBot/Domain/Casino/Games/RockPaperScissors/RockPaperScissors.cs index b45dbd34..3dd73647 100644 --- a/DiscordBot/Domain/Casino/Games/RockPaperScissors/RockPaperScissors.cs +++ b/DiscordBot/Domain/Casino/Games/RockPaperScissors/RockPaperScissors.cs @@ -74,13 +74,13 @@ public override GamePlayerResult GetPlayerGameResult(GamePlayer player) return playerWins ? GamePlayerResult.Won : GamePlayerResult.Lost; } - public override long CalculatePayout(GamePlayer player, ulong totalPot) + public override long CalculatePayout(GamePlayer player, long totalPot) { return player.Result switch { - GamePlayerResult.Won => (long)totalPot - (long)player.Bet, // Winner gets the total pot minus their bet - GamePlayerResult.Lost => -(long)player.Bet, // Loser loses their bet - GamePlayerResult.Tie => 0, // Tie gets bet back (no loss, no gain) + GamePlayerResult.Won => totalPot - player.Bet, + GamePlayerResult.Lost => -player.Bet, + GamePlayerResult.Tie => 0, _ => 0 }; } diff --git a/DiscordBot/Domain/ProfileData.cs b/DiscordBot/Domain/ProfileData.cs index 82ef6162..2942a5fc 100644 --- a/DiscordBot/Domain/ProfileData.cs +++ b/DiscordBot/Domain/ProfileData.cs @@ -7,15 +7,15 @@ public class ProfileData public ulong UserId { get; set; } public string Nickname { get; set; } public string Username { get; set; } - public uint XpTotal { get; set; } - public uint XpRank { get; set; } - public uint KarmaRank { get; set; } - public uint Karma { get; set; } - public uint Level { get; set; } + public long XpTotal { get; set; } + public long XpRank { get; set; } + public long KarmaRank { get; set; } + public int Karma { get; set; } + public int Level { get; set; } public double XpLow { get; set; } public double XpHigh { get; set; } - public uint XpShown { get; set; } - public uint MaxXpShown { get; set; } + public int XpShown { get; set; } + public int MaxXpShown { get; set; } public float XpPercentage { get; set; } public Color MainRoleColor { get; set; } public MagickImage Picture { get; set; } diff --git a/DiscordBot/Extensions/CasinoRepository.cs b/DiscordBot/Extensions/CasinoRepository.cs index 8e8c1c09..ef81e5fc 100644 --- a/DiscordBot/Extensions/CasinoRepository.cs +++ b/DiscordBot/Extensions/CasinoRepository.cs @@ -19,10 +19,10 @@ public interface ICasinoRepo Task> GetTopTokenHolders(int limit); [Sql($"UPDATE {CasinoProps.CasinoTableName} SET {CasinoProps.Tokens} = @tokens, {CasinoProps.UpdatedAt} = @updatedAt WHERE {CasinoProps.UserID} = @userId")] - Task UpdateTokens(string userId, ulong tokens, DateTime updatedAt); + Task UpdateTokens(string userId, long tokens, DateTime updatedAt); [Sql($"UPDATE {CasinoProps.CasinoTableName} SET {CasinoProps.Tokens} = @tokens, {CasinoProps.UpdatedAt} = @updatedAt, {CasinoProps.LastDailyReward} = @lastDailyReward WHERE {CasinoProps.UserID} = @userId")] - Task UpdateTokensAndDailyReward(string userId, ulong tokens, DateTime updatedAt, DateTime lastDailyReward); + Task UpdateTokensAndDailyReward(string userId, long tokens, DateTime updatedAt, DateTime lastDailyReward); [Sql($"DELETE FROM {CasinoProps.CasinoTableName} WHERE {CasinoProps.UserID} = @userId")] Task DeleteCasinoUser(string userId); diff --git a/DiscordBot/Extensions/UserDBRepository.cs b/DiscordBot/Extensions/UserDBRepository.cs index 81dcd90c..ce932b21 100644 --- a/DiscordBot/Extensions/UserDBRepository.cs +++ b/DiscordBot/Extensions/UserDBRepository.cs @@ -6,13 +6,13 @@ public class ServerUser { // ReSharper disable once InconsistentNaming public string UserID { get; set; } - public uint Karma { get; set; } - public uint KarmaWeekly { get; set; } - public uint KarmaMonthly { get; set; } - public uint KarmaYearly { get; set; } - public uint KarmaGiven { get; set; } - public ulong Exp { get; set; } - public uint Level { get; set; } + public int Karma { get; set; } + public int KarmaWeekly { get; set; } + public int KarmaMonthly { get; set; } + public int KarmaYearly { get; set; } + public int KarmaGiven { get; set; } + public long Exp { get; set; } + public int Level { get; set; } // DefaultCity - Optional Location for Weather, BDay, Temp, Time, etc. (Added - Jan 2024) public string DefaultCity { get; set; } = string.Empty; } @@ -60,24 +60,24 @@ public interface IServerUserRepo [Sql($"SELECT {UserProps.UserID}, {UserProps.KarmaYearly} FROM {UserProps.TableName} ORDER BY {UserProps.KarmaYearly} DESC, RANDOM() LIMIT @n")] Task> GetTopKarmaYearly(int n); [Sql($"SELECT COUNT({UserProps.UserID})+1 FROM {UserProps.TableName} WHERE {UserProps.Level} > @level")] - Task GetLevelRank(string userId, uint level); + Task GetLevelRank(string userId, int level); [Sql($"SELECT COUNT({UserProps.UserID})+1 FROM {UserProps.TableName} WHERE {UserProps.Karma} > @karma")] - Task GetKarmaRank(string userId, uint karma); + Task GetKarmaRank(string userId, int karma); #endregion // Ranks #region Update Values [Sql($"UPDATE {UserProps.TableName} SET {UserProps.Karma} = @karma WHERE {UserProps.UserID} = @userId")] - Task UpdateKarma(string userId, uint karma); + Task UpdateKarma(string userId, int karma); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.Karma} = {UserProps.Karma} + 1, {UserProps.KarmaWeekly} = {UserProps.KarmaWeekly} + 1, {UserProps.KarmaMonthly} = {UserProps.KarmaMonthly} + 1, {UserProps.KarmaYearly} = {UserProps.KarmaYearly} + 1 WHERE {UserProps.UserID} = @userId")] Task IncrementKarma(string userId); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.KarmaGiven} = @karmaGiven WHERE {UserProps.UserID} = @userId")] - Task UpdateKarmaGiven(string userId, uint karmaGiven); + Task UpdateKarmaGiven(string userId, int karmaGiven); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.Exp} = @xp WHERE {UserProps.UserID} = @userId")] - Task UpdateXp(string userId, ulong xp); + Task UpdateXp(string userId, long xp); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.Level} = @level WHERE {UserProps.UserID} = @userId")] - Task UpdateLevel(string userId, uint level); + Task UpdateLevel(string userId, int level); [Sql($"UPDATE {UserProps.TableName} SET {UserProps.DefaultCity} = @city WHERE {UserProps.UserID} = @userId")] Task UpdateDefaultCity(string userId, string city); @@ -86,13 +86,13 @@ public interface IServerUserRepo #region Get Single Values [Sql($"SELECT {UserProps.Karma} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] - Task GetKarma(string userId); + Task GetKarma(string userId); [Sql($"SELECT {UserProps.KarmaGiven} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] - Task GetKarmaGiven(string userId); + Task GetKarmaGiven(string userId); [Sql($"SELECT {UserProps.Exp} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] - Task GetXp(string userId); + Task GetXp(string userId); [Sql($"SELECT {UserProps.Level} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] - Task GetLevel(string userId); + Task GetLevel(string userId); [Sql($"SELECT {UserProps.DefaultCity} FROM {UserProps.TableName} WHERE {UserProps.UserID} = @userId")] Task GetDefaultCity(string userId); diff --git a/DiscordBot/Modules/Casino/CasinoSlashModule.Games.cs b/DiscordBot/Modules/Casino/CasinoSlashModule.Games.cs index 6ccf6784..c0d6926d 100644 --- a/DiscordBot/Modules/Casino/CasinoSlashModule.Games.cs +++ b/DiscordBot/Modules/Casino/CasinoSlashModule.Games.cs @@ -300,7 +300,7 @@ public async Task RemoveAIPlayer(string id) #region Betting Actions [ComponentInteraction("bet_add:*:*", true)] - public async Task BetAdd(string id, ulong amount) + public async Task BetAdd(string id, long amount) { await DeferAsync(); @@ -320,7 +320,7 @@ public async Task BetAdd(string id, ulong amount) } [ComponentInteraction("bet_set:*:*", true)] - public async Task BetSet(string id, ulong amount) + public async Task BetSet(string id, long amount) { if (!Context.Interaction.HasResponded) await DeferAsync(); diff --git a/DiscordBot/Modules/Casino/CasinoSlashModule.cs b/DiscordBot/Modules/Casino/CasinoSlashModule.cs index d9ad21db..9d233935 100644 --- a/DiscordBot/Modules/Casino/CasinoSlashModule.cs +++ b/DiscordBot/Modules/Casino/CasinoSlashModule.cs @@ -79,7 +79,7 @@ public async Task CheckTokens() [SlashCommand("gift", "Gift tokens to another user")] public async Task GiftTokens( [Summary("user", "User to gift tokens to")] SocketGuildUser targetUser, - [Summary("amount", "Amount of tokens to gift")] uint amount) + [Summary("amount", "Amount of tokens to gift")] int amount) { if (!await CheckChannelPermissions()) return; @@ -462,7 +462,7 @@ private string CapitalizeFirst(string input) [RequireUserPermission(GuildPermission.Administrator)] public async Task SetTokens( [Summary("user", "User to set tokens for")] SocketGuildUser targetUser, - [Summary("amount", "New token amount")] uint amount) + [Summary("amount", "New token amount")] int amount) { if (!await CheckChannelPermissions()) return; @@ -484,13 +484,13 @@ public async Task SetTokens( [RequireUserPermission(GuildPermission.Administrator)] public async Task AddTokens( [Summary("user", "User to add tokens to")] SocketGuildUser targetUser, - [Summary("amount", "Amount of tokens to add")] uint amount) + [Summary("amount", "Amount of tokens to add")] int amount) { if (!await CheckChannelPermissions()) return; await Context.Interaction.DeferAsync(ephemeral: true); - await CasinoService.UpdateUserTokens(targetUser.Id.ToString(), (long)amount, TransactionType.Admin, new Dictionary + await CasinoService.UpdateUserTokens(targetUser.Id.ToString(), amount, TransactionType.Admin, new Dictionary { ["admin"] = Context.User.Id.ToString(), ["action"] = "add" diff --git a/DiscordBot/Modules/UserModule.cs b/DiscordBot/Modules/UserModule.cs index ced5b0c4..d8bc3b65 100644 --- a/DiscordBot/Modules/UserModule.cs +++ b/DiscordBot/Modules/UserModule.cs @@ -251,7 +251,7 @@ public async Task UserCompleted(string message) return; } - const uint xpGain = 5000; + const int xpGain = 5000; var userXp = await DatabaseService.Query.GetXp(userId.ToString()); await DatabaseService.Query.UpdateXp(userId.ToString(), userXp + xpGain); await Context.Message.DeleteAsync(); @@ -468,7 +468,7 @@ public async Task TopKarmaYearly() await ReplyAsync(embed: embed).DeleteAfterTime(minutes: 1); } - private async Task GenerateRankEmbedFromList(List<(ulong userID, uint value)> data, string labelName) + private async Task GenerateRankEmbedFromList(List<(ulong userID, int value)> data, string labelName) { var embedBuilder = new EmbedBuilder { diff --git a/DiscordBot/Services/Casino/CasinoService.cs b/DiscordBot/Services/Casino/CasinoService.cs index 2550cef6..2cecfb2e 100644 --- a/DiscordBot/Services/Casino/CasinoService.cs +++ b/DiscordBot/Services/Casino/CasinoService.cs @@ -39,7 +39,7 @@ public async Task GetOrCreateCasinoUser(string userId) }; var createdUser = await _databaseService.CasinoQuery.InsertCasinoUser(newUser); - await RecordTransaction(userId, (long)_settings.CasinoStartingTokens, TransactionType.TokenInitialisation); + await RecordTransaction(userId, _settings.CasinoStartingTokens, TransactionType.TokenInitialisation); await _loggingService.LogChannelAndFile($"{ServiceName}: Created new casino user {userId} with {_settings.CasinoStartingTokens} starting tokens"); return createdUser; } @@ -51,7 +51,7 @@ public async Task GetOrCreateCasinoUser(string userId) } } - public async Task TransferTokens(string fromUserId, string toUserId, ulong amount) + public async Task TransferTokens(string fromUserId, string toUserId, long amount) { var fromUser = await GetOrCreateCasinoUser(fromUserId); var toUser = await GetOrCreateCasinoUser(toUserId); @@ -64,11 +64,11 @@ public async Task TransferTokens(string fromUserId, string toUserId, ulong await _databaseService.CasinoQuery.UpdateTokens(toUserId, toUser.Tokens + amount, DateTime.UtcNow); // Record transactions - await RecordTransaction(fromUserId, -(long)amount, TransactionType.Gift, new Dictionary + await RecordTransaction(fromUserId, -amount, TransactionType.Gift, new Dictionary { ["to"] = toUserId, }); - await RecordTransaction(toUserId, (long)amount, TransactionType.Gift, new Dictionary + await RecordTransaction(toUserId, amount, TransactionType.Gift, new Dictionary { ["from"] = fromUserId }); @@ -81,11 +81,11 @@ public async Task UpdateUserTokens(string userId, long deltaTokens, TransactionT try { var user = await GetOrCreateCasinoUser(userId); - var newBalance = (long)user.Tokens + deltaTokens; + var newBalance = user.Tokens + deltaTokens; // Prevent negative balance if (newBalance < 0) newBalance = 0; - await _databaseService.CasinoQuery.UpdateTokens(userId, (ulong)newBalance, DateTime.UtcNow); + await _databaseService.CasinoQuery.UpdateTokens(userId, newBalance, DateTime.UtcNow); await RecordTransaction(userId, deltaTokens, transactionType, details); } catch (Exception ex) @@ -96,11 +96,11 @@ public async Task UpdateUserTokens(string userId, long deltaTokens, TransactionT } } - public async Task SetUserTokens(string userId, ulong amount, string adminUserId) + public async Task SetUserTokens(string userId, long amount, string adminUserId) { await _databaseService.CasinoQuery.UpdateTokens(userId, amount, DateTime.UtcNow); - await RecordTransaction(userId, (long)amount, TransactionType.Admin, new Dictionary + await RecordTransaction(userId, amount, TransactionType.Admin, new Dictionary { ["admin"] = adminUserId, ["action"] = "set" @@ -315,7 +315,7 @@ public bool IsChannelAllowed(ulong channelId) #region Daily Rewards - public async Task<(bool success, ulong tokensAwarded, ulong newBalance, DateTime nextRewardTime)> TryClaimDailyReward(string userId) + public async Task<(bool success, long tokensAwarded, long newBalance, DateTime nextRewardTime)> TryClaimDailyReward(string userId) { try { @@ -332,7 +332,7 @@ public bool IsChannelAllowed(ulong channelId) var tokensAwarded = _settings.CasinoDailyRewardTokens; var newBalance = user.Tokens + tokensAwarded; await _databaseService.CasinoQuery.UpdateTokensAndDailyReward(userId, newBalance, now, now); - await RecordTransaction(userId, (long)tokensAwarded, TransactionType.DailyReward); + await RecordTransaction(userId, tokensAwarded, TransactionType.DailyReward); await _loggingService.LogChannelAndFile($"{ServiceName}: User {userId} claimed daily reward of {tokensAwarded} tokens"); return (true, tokensAwarded, newBalance, now.AddSeconds(_settings.CasinoDailyRewardIntervalSeconds)); diff --git a/DiscordBot/Services/Casino/GameService.cs b/DiscordBot/Services/Casino/GameService.cs index a794ef0c..3f4d8b2c 100644 --- a/DiscordBot/Services/Casino/GameService.cs +++ b/DiscordBot/Services/Casino/GameService.cs @@ -73,7 +73,7 @@ public async Task JoinGame(IDiscordGameSession session, ulong userId) session.AddPlayer(userId, 1); } - public async Task SetBet(IDiscordGameSession session, ulong userId, ulong bet) + public async Task SetBet(IDiscordGameSession session, ulong userId, long bet) { var user = await _casinoService.GetOrCreateCasinoUser(userId.ToString()); if (bet > user.Tokens) throw new InvalidOperationException("You do not have enough tokens."); diff --git a/DiscordBot/Services/UserService.cs b/DiscordBot/Services/UserService.cs index f865581a..fbfc169b 100644 --- a/DiscordBot/Services/UserService.cs +++ b/DiscordBot/Services/UserService.cs @@ -263,7 +263,7 @@ public async Task UpdateXp(SocketMessage messageParam) var xpGain = (int)Math.Round((baseXp + bonusXp) * reduceXp); - await _databaseService.Query.UpdateXp(userId.ToString(), user.Exp + (uint)xpGain); + await _databaseService.Query.UpdateXp(userId.ToString(), user.Exp + (long)xpGain); _loggingService.LogXp(messageParam.Channel.Name, messageParam.Author.Username, baseXp, bonusXp, reduceXp, xpGain); @@ -300,9 +300,9 @@ private async Task LevelUp(SocketMessage messageParam, ulong userId) //TODO Add level up card } - private double GetXpLow(uint level) => 70d - 139.5d * (level + 1d) + 69.5 * Math.Pow(level + 1d, 2d); + private double GetXpLow(int level) => 70d - 139.5d * (level + 1d) + 69.5 * Math.Pow(level + 1d, 2d); - private double GetXpHigh(uint level) => 70d - 139.5d * (level + 2d) + 69.5 * Math.Pow(level + 2d, 2d); + private double GetXpHigh(int level) => 70d - 139.5d * (level + 2d) + 69.5 * Math.Pow(level + 2d, 2d); private SkinData GetSkinData() => JsonConvert.DeserializeObject(File.ReadAllText($"{_settings.AssetsRootPath}/skins/skin.json"), @@ -333,8 +333,8 @@ public async Task GenerateProfileCard(IUser user) var xpLow = GetXpLow(level); var xpHigh = GetXpHigh(level); - var xpShown = (uint)(xpTotal - xpLow); - var maxXpShown = (uint)(xpHigh - xpLow); + var xpShown = (int)(xpTotal - xpLow); + var maxXpShown = (int)(xpHigh - xpLow); var percentage = (float)xpShown / maxXpShown; @@ -355,8 +355,8 @@ public async Task GenerateProfileCard(IUser user) var profile = new ProfileData { Karma = karma, - KarmaRank = (uint)karmaRank, - Level = (uint)level, + KarmaRank = karmaRank, + Level = level, MainRoleColor = mainRole.Color, MaxXpShown = maxXpShown, Nickname = ((IGuildUser)user).Nickname, @@ -365,9 +365,9 @@ public async Task GenerateProfileCard(IUser user) XpHigh = xpHigh, XpLow = xpLow, XpPercentage = percentage, - XpRank = (uint)xpRank, + XpRank = xpRank, XpShown = xpShown, - XpTotal = (uint)xpTotal + XpTotal = xpTotal }; var background = new MagickImage($"{_settings.AssetsRootPath}/skins/{skin.Background}"); diff --git a/DiscordBot/Settings/Deserialized/Settings.cs b/DiscordBot/Settings/Deserialized/Settings.cs index c751ccff..9b6be99e 100644 --- a/DiscordBot/Settings/Deserialized/Settings.cs +++ b/DiscordBot/Settings/Deserialized/Settings.cs @@ -139,12 +139,12 @@ public class BotSettings #region Casino Settings public bool CasinoEnabled { get; set; } = true; - public ulong CasinoStartingTokens { get; set; } = 1000; + public long CasinoStartingTokens { get; set; } = 1000; public List CasinoAllowedChannels { get; set; } = new List(); public int CasinoGameTimeoutMinutes { get; set; } = 5; // Daily Reward Settings - public ulong CasinoDailyRewardTokens { get; set; } = 100; + public long CasinoDailyRewardTokens { get; set; } = 100; public int CasinoDailyRewardIntervalSeconds { get; set; } = 86400; // 24 hours = 86400 seconds #endregion // Casino Settings From ef5497c52ea6a1e49eb555cb6dbe4980a88b0d11 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 22:05:36 +0200 Subject: [PATCH 14/25] fix(bot): update bot image to latest version 17fc156 --- k8s/dev/bot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/dev/bot.yaml b/k8s/dev/bot.yaml index d848126f..4f45e86e 100644 --- a/k8s/dev/bot.yaml +++ b/k8s/dev/bot.yaml @@ -128,7 +128,7 @@ spec: - ALL containers: - name: bot - image: ghcr.io/unity-developer-community/udc-bot-dev:832a135 + image: ghcr.io/unity-developer-community/udc-bot-dev:17fc156 volumeMounts: - name: app-settings mountPath: /app/Settings From f30e121919132b76662b0f7b2131aecd48db160a Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 22:07:17 +0200 Subject: [PATCH 15/25] feat(adminer): add deployment, service, and ingress for adminer for dev Co-authored-by: Copilot --- k8s/dev/adminer.yaml | 95 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 k8s/dev/adminer.yaml diff --git a/k8s/dev/adminer.yaml b/k8s/dev/adminer.yaml new file mode 100644 index 00000000..81bb0fcf --- /dev/null +++ b/k8s/dev/adminer.yaml @@ -0,0 +1,95 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: adminer + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: adminer + template: + metadata: + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: dev + spec: + containers: + - name: adminer + image: adminer:4 + ports: + - containerPort: 8080 + protocol: TCP + env: + - name: ADMINER_DEFAULT_SERVER + value: postgresql + - name: ADMINER_DESIGN + value: dracula + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + name: adminer + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + selector: + app.kubernetes.io/name: adminer +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: adminer + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: dev + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: traefik + tls: + - hosts: + - adminer.dev.bot.udc.ovh + secretName: adminer-dev-tls + rules: + - host: adminer.dev.bot.udc.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: adminer + port: + number: 80 From e9c7811e2439424e84f521e4704b92148400662a Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 22:11:35 +0200 Subject: [PATCH 16/25] Added adminer for prod and docker-compose Co-authored-by: Copilot --- docker-compose.yml | 11 +++++ k8s/prod/adminer.yaml | 96 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 k8s/prod/adminer.yaml diff --git a/docker-compose.yml b/docker-compose.yml index c9c291d1..d72e2a9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,5 +43,16 @@ services: - db restart: always + adminer: + image: adminer:4 + depends_on: + - db + restart: always + ports: + - 8081:8080 + environment: + ADMINER_DEFAULT_SERVER: db + ADMINER_DESIGN: dracula + volumes: db_data: diff --git a/k8s/prod/adminer.yaml b/k8s/prod/adminer.yaml new file mode 100644 index 00000000..b4655e2b --- /dev/null +++ b/k8s/prod/adminer.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: adminer + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: adminer + template: + metadata: + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: prod + spec: + containers: + - name: adminer + image: adminer:4 + ports: + - containerPort: 8080 + protocol: TCP + env: + - name: ADMINER_DEFAULT_SERVER + value: postgresql + - name: ADMINER_DESIGN + value: dracula + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + name: adminer + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: prod +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + selector: + app.kubernetes.io/name: adminer +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: adminer + namespace: udc-bot-prod + labels: + app.kubernetes.io/name: adminer + app.kubernetes.io/part-of: udc-bot + environment: prod + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - adminer.bot.udc.ovh + secretName: adminer-prod-tls + rules: + - host: adminer.bot.udc.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: adminer + port: + number: 80 From 0c0f359be8a92025ac24cb30b62d19fe77485bd5 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Fri, 3 Apr 2026 22:23:09 +0200 Subject: [PATCH 17/25] Removed pgadmin --- .gitignore | 3 - docker-compose.yml | 29 +----- k8s/dev/external-secrets.yaml | 19 ---- k8s/dev/pgadmin.yaml | 161 -------------------------------- k8s/prod/external-secrets.yaml | 19 ---- k8s/prod/pgadmin.yaml | 162 --------------------------------- pgadmin/servers.json | 14 --- 7 files changed, 4 insertions(+), 403 deletions(-) delete mode 100644 k8s/dev/pgadmin.yaml delete mode 100644 k8s/prod/pgadmin.yaml delete mode 100644 pgadmin/servers.json diff --git a/.gitignore b/.gitignore index 0fd898e5..9908e11e 100644 --- a/.gitignore +++ b/.gitignore @@ -455,6 +455,3 @@ fabric.properties # Bot runtime-generated data (SERVER/ is created by the bot at runtime) DiscordBot/SERVER/ - -# pgAdmin pgpass contains plaintext credentials -pgadmin/pgpass diff --git a/docker-compose.yml b/docker-compose.yml index d72e2a9c..f392ab0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,26 +12,16 @@ services: POSTGRES_USER: udcbot POSTGRES_PASSWORD: 123456789 - pgadmin: - image: dpage/pgadmin4 + adminer: + image: adminer:4 depends_on: - db restart: always ports: - 8080:80 environment: - PGADMIN_DEFAULT_EMAIL: admin@local.dev - PGADMIN_DEFAULT_PASSWORD: admin - PGADMIN_SERVER_JSON_FILE: /pgadmin4/servers.json - volumes: - - ./pgadmin/servers.json:/pgadmin4/servers.json:ro - - ./pgadmin/pgpass:/pgpass_src:ro - entrypoint: > - /bin/sh -c " - cp /pgpass_src /tmp/pgpass && - chmod 600 /tmp/pgpass && - /entrypoint.sh - " + ADMINER_DEFAULT_SERVER: db + ADMINER_DESIGN: dracula bot: build: . @@ -43,16 +33,5 @@ services: - db restart: always - adminer: - image: adminer:4 - depends_on: - - db - restart: always - ports: - - 8081:8080 - environment: - ADMINER_DEFAULT_SERVER: db - ADMINER_DESIGN: dracula - volumes: db_data: diff --git a/k8s/dev/external-secrets.yaml b/k8s/dev/external-secrets.yaml index 7f94fa6b..c7f00e3f 100644 --- a/k8s/dev/external-secrets.yaml +++ b/k8s/dev/external-secrets.yaml @@ -37,25 +37,6 @@ spec: key: "Bot Token - Dev" property: identifiant --- -# pgAdmin login password — from 1Password "pgAdmin - Dev" -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: pgadmin-credentials - namespace: udc-bot-dev -spec: - refreshInterval: 1h - secretStoreRef: - name: onepassword - kind: ClusterSecretStore - target: - name: pgadmin-credentials - data: - - secretKey: password - remoteRef: - key: "pgAdmin - Dev" - property: password ---- # Bot API keys — from various 1Password items apiVersion: external-secrets.io/v1 kind: ExternalSecret diff --git a/k8s/dev/pgadmin.yaml b/k8s/dev/pgadmin.yaml deleted file mode 100644 index 3845bbf8..00000000 --- a/k8s/dev/pgadmin.yaml +++ /dev/null @@ -1,161 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: pgadmin-servers - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev -data: - servers.json: | - { - "Servers": { - "1": { - "Name": "UDC Bot - PostgreSQL (Dev)", - "Group": "Servers", - "Host": "postgresql", - "Port": 5432, - "MaintenanceDB": "udcbot", - "Username": "udcbot", - "SSLMode": "prefer", - "PassFile": "/tmp/pgpass" - } - } - } ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgadmin - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: pgadmin - template: - metadata: - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev - spec: - initContainers: - - name: create-pgpass - image: busybox:1.36 - command: - - sh - - -c - - | - echo "postgresql:5432:udcbot:udcbot:${PGPASSWORD}" > /pgpass/pgpass - chmod 600 /pgpass/pgpass - env: - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: postgresql-credentials - key: password - volumeMounts: - - name: pgpass - mountPath: /pgpass - securityContext: - allowPrivilegeEscalation: false - containers: - - name: pgadmin - image: dpage/pgadmin4:8.14 - ports: - - containerPort: 80 - protocol: TCP - env: - - name: PGADMIN_DEFAULT_EMAIL - value: admin@udc.ovh - - name: PGADMIN_DEFAULT_PASSWORD - valueFrom: - secretKeyRef: - name: pgadmin-credentials - key: password - - name: PGADMIN_SERVER_JSON_FILE - value: /pgadmin4/servers.json - volumeMounts: - - name: servers-config - mountPath: /pgadmin4/servers.json - subPath: servers.json - readOnly: true - - name: pgpass - mountPath: /tmp/pgpass - subPath: pgpass - readOnly: true - resources: - requests: - cpu: 25m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - readinessProbe: - httpGet: - path: /misc/ping - port: 80 - initialDelaySeconds: 10 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - volumes: - - name: servers-config - configMap: - name: pgadmin-servers - - name: pgpass - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: pgadmin - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: 80 - protocol: TCP - selector: - app.kubernetes.io/name: pgadmin ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: pgadmin - namespace: udc-bot-dev - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: dev - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: traefik - tls: - - hosts: - - pgadmin.dev.bot.udc.ovh - secretName: pgadmin-dev-tls - rules: - - host: pgadmin.dev.bot.udc.ovh - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: pgadmin - port: - number: 80 diff --git a/k8s/prod/external-secrets.yaml b/k8s/prod/external-secrets.yaml index ec546195..05d4c55c 100644 --- a/k8s/prod/external-secrets.yaml +++ b/k8s/prod/external-secrets.yaml @@ -37,25 +37,6 @@ spec: key: "Bot Token - Prod" property: identifiant --- -# pgAdmin login password — from 1Password "pgAdmin - Prod" -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: pgadmin-credentials - namespace: udc-bot-prod -spec: - refreshInterval: 1h - secretStoreRef: - name: onepassword - kind: ClusterSecretStore - target: - name: pgadmin-credentials - data: - - secretKey: password - remoteRef: - key: "pgAdmin - Prod" - property: password ---- # Bot API keys — same keys shared between dev and prod apiVersion: external-secrets.io/v1 kind: ExternalSecret diff --git a/k8s/prod/pgadmin.yaml b/k8s/prod/pgadmin.yaml deleted file mode 100644 index edac4f38..00000000 --- a/k8s/prod/pgadmin.yaml +++ /dev/null @@ -1,162 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: pgadmin-servers - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod -data: - servers.json: | - { - "Servers": { - "1": { - "Name": "UDC Bot - PostgreSQL (Prod)", - "Group": "Servers", - "Host": "postgresql", - "Port": 5432, - "MaintenanceDB": "udcbot", - "Username": "udcbot", - "SSLMode": "prefer", - "PassFile": "/tmp/pgpass" - } - } - } ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgadmin - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: pgadmin - template: - metadata: - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod - spec: - initContainers: - - name: create-pgpass - image: busybox:1.36 - command: - - sh - - -c - - | - echo "postgresql:5432:udcbot:udcbot:${PGPASSWORD}" > /pgpass/pgpass - chmod 600 /pgpass/pgpass - env: - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: postgresql-credentials - key: password - volumeMounts: - - name: pgpass - mountPath: /pgpass - securityContext: - allowPrivilegeEscalation: false - containers: - - name: pgadmin - image: dpage/pgadmin4:8.14 - ports: - - containerPort: 80 - protocol: TCP - env: - - name: PGADMIN_DEFAULT_EMAIL - value: admin@udc.ovh - - name: PGADMIN_DEFAULT_PASSWORD - valueFrom: - secretKeyRef: - name: pgadmin-credentials - key: password - - name: PGADMIN_SERVER_JSON_FILE - value: /pgadmin4/servers.json - volumeMounts: - - name: servers-config - mountPath: /pgadmin4/servers.json - subPath: servers.json - readOnly: true - - name: pgpass - mountPath: /tmp/pgpass - subPath: pgpass - readOnly: true - resources: - requests: - cpu: 25m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - readinessProbe: - httpGet: - path: /misc/ping - port: 80 - initialDelaySeconds: 10 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - volumes: - - name: servers-config - configMap: - name: pgadmin-servers - - name: pgpass - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: pgadmin - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: 80 - protocol: TCP - selector: - app.kubernetes.io/name: pgadmin ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: pgadmin - namespace: udc-bot-prod - labels: - app.kubernetes.io/name: pgadmin - app.kubernetes.io/part-of: udc-bot - environment: prod - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/router.middlewares: default-ip-allowlist@kubernetescrd -spec: - ingressClassName: traefik - tls: - - hosts: - - pgadmin.bot.udc.ovh - secretName: pgadmin-prod-tls - rules: - - host: pgadmin.bot.udc.ovh - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: pgadmin - port: - number: 80 diff --git a/pgadmin/servers.json b/pgadmin/servers.json deleted file mode 100644 index 2bcccea1..00000000 --- a/pgadmin/servers.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Servers": { - "1": { - "Name": "UDC Bot - PostgreSQL", - "Group": "Servers", - "Host": "db", - "Port": 5432, - "MaintenanceDB": "udcbot", - "Username": "udcbot", - "SSLMode": "prefer", - "PassFile": "/tmp/pgpass" - } - } -} From 1ac55cde98bf34b05ac449e24723ea45d572069e Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Sat, 4 Apr 2026 01:25:13 +0200 Subject: [PATCH 18/25] feat(migration): add MySQL to PostgreSQL migration plan Co-authored-by: Copilot --- .../data-migration-mysql-to-postgresql.md | 248 ++++++++++++++++++ k8s/dev/external-secrets.yaml | 40 +++ k8s/dev/mysql.yaml | 117 +++++++++ k8s/dev/pgloader-migration.yaml | 109 ++++++++ k8s/prod/external-secrets.yaml | 40 +++ k8s/prod/mysql.yaml | 117 +++++++++ k8s/prod/pgloader-migration.yaml | 110 ++++++++ 7 files changed, 781 insertions(+) create mode 100644 docs/plans/data-migration-mysql-to-postgresql.md create mode 100644 k8s/dev/mysql.yaml create mode 100644 k8s/dev/pgloader-migration.yaml create mode 100644 k8s/prod/mysql.yaml create mode 100644 k8s/prod/pgloader-migration.yaml diff --git a/docs/plans/data-migration-mysql-to-postgresql.md b/docs/plans/data-migration-mysql-to-postgresql.md new file mode 100644 index 00000000..e053135f --- /dev/null +++ b/docs/plans/data-migration-mysql-to-postgresql.md @@ -0,0 +1,248 @@ +# MySQL → PostgreSQL Data Migration Plan + +## Overview + +Migrate production data from MySQL (`udc-bot-prod`) to PostgreSQL. The schema has already been migrated (DDL in `DatabaseService.cs`). This plan covers **data migration and the production cutover strategy**. + +## Cutover Strategy + +The PR (`feature/postgre`) replaces MySQL with PostgreSQL in k8s manifests. This means we **lose MySQL** once the new manifests are deployed. The migration must follow this sequence: + +### Phase 0: Test on Dev (current step) + +**Goal:** Validate the pgloader migration from prod MySQL → dev PostgreSQL before touching prod. + +1. **Scale down dev bot** — `kubectl scale deployment/udc-bot -n udc-bot-dev --replicas=0` +2. **Deploy the pgloader Job** — `kubectl apply -f k8s/dev/pgloader-migration.yaml` + - Creates a temporary ExternalSecret for MySQL prod credentials in dev namespace + - Runs pgloader in `data only` mode (tables already exist from bot startup) + - Truncates existing dev data, imports prod data, resets sequences +3. **Check Job logs** — `kubectl logs job/mysql-to-postgresql-migration -n udc-bot-dev` +4. **Verify data** — connect via Adminer or `kubectl exec` and check row counts +5. **Scale dev bot back up** — `kubectl scale deployment/udc-bot -n udc-bot-dev --replicas=1` +6. **Test bot commands** — `!profile`, `thanks @someone`, casino commands +7. **Clean up** — delete the Job and temporary MySQL secret: + ```bash + kubectl delete job mysql-to-postgresql-migration -n udc-bot-dev + kubectl delete externalsecret mysql-prod-credentials -n udc-bot-dev + kubectl delete secret mysql-prod-credentials -n udc-bot-dev + ``` + +**File:** `k8s/dev/pgloader-migration.yaml` (ExternalSecret + ConfigMap + Job) + +### Phase 1: Prepare (before merging PR) + +1. **Deploy PostgreSQL alongside MySQL in prod** — add `postgresql.yaml` to `k8s/prod/` while keeping `mysql.yaml` intact. Both databases run simultaneously. +2. **Scale down the bot** — `kubectl scale deployment/udc-bot -n udc-bot-prod --replicas=0` — freeze writes to MySQL. +3. **Backup MySQL** — `mysqldump` as safety net. + +### Phase 2: Migrate (both databases running) + +4. **Run pgloader Job** — streams data from MySQL → PostgreSQL within the cluster. +5. **Verify data** — row counts, spot-checks, sequence values. + +### Phase 3: Switch (controlled cutover) + +6. **Deploy the new bot** (from `feature/postgre`) pointing to PostgreSQL. +7. **Verify bot commands** — `!profile`, karma, casino. +8. **Keep MySQL running** for 24-48h as rollback safety net. + +### Phase 4: Cleanup + +9. **Remove MySQL** from prod k8s manifests (`mysql.yaml`, ExternalSecrets). +10. **Remove MySQL 1Password items** if no longer needed. +11. **Move this plan** to `docs/plans/done/`. + +## Data Inventory + +| Table | Rows | Key Columns | Notes | +|-------|------|-------------|-------| +| `users` | **54,302** | `ID` (auto-inc), `UserID` (varchar PK) | Karma, XP, Level data | +| `casino_users` | **22** | `Id` (auto-inc), `UserID` (varchar unique) | Token balances | +| `token_transactions` | **1,441** | `Id` (auto-inc), `UserID`, `Amount`, `Type` | Transaction history | + +### Schema Differences + +| MySQL Type | PostgreSQL Type | Affected Columns | +|-----------|----------------|-----------------| +| `int unsigned` | `integer` | Karma*, KarmaGiven, Level, Id | +| `bigint unsigned` | `bigint` | Exp, Tokens | +| `timestamp` (MySQL default) | `timestamp` | CreatedAt, UpdatedAt, LastDailyReward | +| `varchar(32)` | `varchar(32)` | UserID (no change) | + +## Migration Strategy + +**Recommended: `pgloader`** — purpose-built MySQL-to-PostgreSQL ETL tool. + +### Why pgloader + +- Handles type mapping automatically (unsigned → signed) +- Streams data (no intermediate files for 54K rows) +- Single command, declarative config +- Handles timestamp conversion, encoding, and NULL differences + +### Alternative: CSV export/import + +Simpler but manual. Good fallback if pgloader isn't available in the cluster. + +--- + +## Execution Plan + +### Prerequisites + +- [ ] PostgreSQL deployed in prod **alongside** MySQL (temporary dual-database state) +- [ ] Bot scaled to 0 replicas (no writes during migration) +- [ ] MySQL backup taken +- [ ] Cross-namespace DNS verified: `mysql.udc-bot-prod.svc.cluster.local` reachable from migration pod + +### Option A: pgloader (Recommended) + +#### Step 1: Create pgloader Job + +Deploy a one-shot Kubernetes Job that runs pgloader with a config targeting MySQL→PostgreSQL. + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: mysql-to-postgresql-migration + namespace: udc-bot-dev +spec: + template: + spec: + containers: + - name: pgloader + image: ghcr.io/dimitri/pgloader:latest + command: + - pgloader + - /config/migration.load + volumeMounts: + - name: config + mountPath: /config + volumes: + - name: config + configMap: + name: pgloader-config + restartPolicy: Never + backoffLimit: 1 +``` + +#### Step 2: pgloader Configuration + +``` +LOAD DATABASE + FROM mysql://root:PASSWORD@mysql.udc-bot-prod.svc.cluster.local:3306/udcbot + INTO postgresql://udcbot:PASSWORD@postgresql.udc-bot-dev.svc.cluster.local:5432/udcbot + +WITH include drop, create tables, create indexes, reset sequences, + workers = 2, concurrency = 1 + +SET maintenance_work_mem to '128MB' + +CAST type int unsigned to integer, + type bigint unsigned to bigint + +-- Only migrate these tables +INCLUDING ONLY TABLE NAMES MATCHING 'users', 'casino_users', 'token_transactions' + +BEFORE LOAD DO + $$ TRUNCATE users, casino_users, token_transactions CASCADE; $$ +; +``` + +#### Step 3: Verify + +```sql +SELECT COUNT(*) FROM users; -- Expect: 54,302 +SELECT COUNT(*) FROM casino_users; -- Expect: 22 +SELECT COUNT(*) FROM token_transactions; -- Expect: 1,441 +``` + +### Option B: CSV Export/Import (Fallback) + +#### Step 1: Export from MySQL + +```bash +# Run inside MySQL pod +mysqldump -u root -p udcbot users casino_users token_transactions \ + --compatible=postgresql --no-create-info --complete-insert \ + --skip-quote-names > /tmp/data.sql +``` + +Or per-table CSV: + +```bash +mysql -u root -p udcbot -e "SELECT * FROM users" --batch > /tmp/users.tsv +mysql -u root -p udcbot -e "SELECT * FROM casino_users" --batch > /tmp/casino_users.tsv +mysql -u root -p udcbot -e "SELECT * FROM token_transactions" --batch > /tmp/token_transactions.tsv +``` + +#### Step 2: Transfer files + +```bash +kubectl cp udc-bot-prod/mysql-pod:/tmp/users.tsv ./users.tsv +kubectl cp udc-bot-prod/mysql-pod:/tmp/casino_users.tsv ./casino_users.tsv +kubectl cp udc-bot-prod/mysql-pod:/tmp/token_transactions.tsv ./token_transactions.tsv +``` + +#### Step 3: Import to PostgreSQL + +```bash +kubectl cp ./users.tsv udc-bot-dev/postgresql-pod:/tmp/users.tsv +kubectl exec -it deployment/postgresql -n udc-bot-dev -- psql -U udcbot -d udcbot -c "\COPY users FROM '/tmp/users.tsv' WITH (FORMAT text, HEADER true)" +``` + +Repeat for each table. + +#### Step 4: Fix sequences + +After import, auto-increment sequences will be out of sync: + +```sql +SELECT setval('users_id_seq', (SELECT MAX(id) FROM users)); +SELECT setval('casino_users_id_seq', (SELECT MAX(id) FROM casino_users)); +SELECT setval('token_transactions_id_seq', (SELECT MAX(id) FROM token_transactions)); +``` + +--- + +## Post-Migration Checklist + +- [ ] Verify row counts match source +- [ ] Spot-check specific users (karma, level, tokens) +- [ ] Test `!profile` with a known user +- [ ] Test `thanks @someone` and verify karma increment +- [ ] Test casino commands +- [ ] Verify auto-increment sequences are correct + +## Risk Mitigation + +- **MySQL stays running** for 24-48h after cutover — instant rollback by redeploying the old bot image +- **MySQL backup** before migration: `mysqldump -u root -p udcbot > backup.sql` +- **PostgreSQL is idempotent**: Tables are created by the bot on startup; migration can be re-run with `TRUNCATE` first +- **Bot is down during migration** (scaled to 0) — no data inconsistency possible +- **Rollback plan**: Scale down new bot → scale up old bot → MySQL is still there with original data + +## K8s Manifest Changes for Cutover + +To run both databases temporarily, you need to **add** `postgresql.yaml` + PostgreSQL ExternalSecret to `k8s/prod/` **before** removing `mysql.yaml`. The PR should be split or the manifests applied in steps: + +1. First ArgoCD sync: Add PostgreSQL manifests (MySQL stays) +2. Run migration Job +3. Second ArgoCD sync: Deploy new bot image + remove MySQL manifests + +## Timeline + +1. Deploy PostgreSQL in prod alongside MySQL (5 min) +2. Scale bot to 0 (1 min) +3. Backup MySQL (5 min) +4. Deploy pgloader Job (5 min) +5. Wait for completion (< 1 min for 54K rows) +6. Verify data (10 min) +7. Deploy new bot pointing to PostgreSQL (5 min) +8. Test bot commands (10 min) +9. Monitor for 24-48h +10. Remove MySQL (5 min) + +**Active work: ~45 minutes** | **Total with monitoring: 24-48h** diff --git a/k8s/dev/external-secrets.yaml b/k8s/dev/external-secrets.yaml index c7f00e3f..521f5389 100644 --- a/k8s/dev/external-secrets.yaml +++ b/k8s/dev/external-secrets.yaml @@ -18,6 +18,46 @@ spec: key: "PostgreSQL Server - Dev" property: password --- +# TEMPORARY: MySQL credentials needed for migration testing. Remove after migration. +# MySQL root password — from 1Password "MySQL Server - Root User - Dev" +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: mysql-credentials + namespace: udc-bot-dev +spec: + refreshInterval: 1h + secretStoreRef: + name: onepassword + kind: ClusterSecretStore + target: + name: mysql-credentials + data: + - secretKey: password + remoteRef: + key: "MySQL Server - Root User - Dev" + property: password +--- +# TEMPORARY: MySQL user credentials needed for migration testing. Remove after migration. +# MySQL udcbot user password — from 1Password "MySQL Server - UDC User - Dev" +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: mysql-user-credentials + namespace: udc-bot-dev +spec: + refreshInterval: 1h + secretStoreRef: + name: onepassword + kind: ClusterSecretStore + target: + name: mysql-user-credentials + data: + - secretKey: password + remoteRef: + key: "MySQL Server - UDC User - Dev" + property: password +--- # Discord bot token — from 1Password "Bot Token - Dev" apiVersion: external-secrets.io/v1 kind: ExternalSecret diff --git a/k8s/dev/mysql.yaml b/k8s/dev/mysql.yaml new file mode 100644 index 00000000..88b0a674 --- /dev/null +++ b/k8s/dev/mysql.yaml @@ -0,0 +1,117 @@ +--- +# TEMPORARY: Keep MySQL running alongside PostgreSQL for migration testing. +# Remove this file after migration is verified. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: mysql + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: mysql + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: mysql + template: + metadata: + labels: + app.kubernetes.io/name: mysql + app.kubernetes.io/part-of: udc-bot + environment: dev + spec: + containers: + - name: mysql + image: mysql:8.0 + ports: + - containerPort: 3306 + protocol: TCP + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-credentials + key: password + - name: MYSQL_DATABASE + value: udcbot + - name: MYSQL_USER + value: udcbot + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-user-credentials + key: password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -h 127.0.0.1 -u root -p"${MYSQL_ROOT_PASSWORD}" + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -h 127.0.0.1 -u root -p"${MYSQL_ROOT_PASSWORD}" + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: mysql + app.kubernetes.io/part-of: udc-bot + environment: dev +spec: + type: ClusterIP + ports: + - port: 3306 + targetPort: 3306 + protocol: TCP + selector: + app.kubernetes.io/name: mysql diff --git a/k8s/dev/pgloader-migration.yaml b/k8s/dev/pgloader-migration.yaml new file mode 100644 index 00000000..d9638a23 --- /dev/null +++ b/k8s/dev/pgloader-migration.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgloader-config + namespace: udc-bot-dev + labels: + app.kubernetes.io/name: pgloader-migration + app.kubernetes.io/part-of: udc-bot + environment: dev +data: + run-migration.sh: | + #!/bin/sh + set -e + + # URL-encode passwords using pure shell (handles special chars like %) + urlencode() { + printf '%s' "$1" | od -An -tx1 | tr ' ' '%' | tr -d '\n' | sed 's/^%//' + } + + MYSQL_PASS_ENCODED=$(urlencode "${MYSQL_PASSWORD}") + PG_PASS_ENCODED=$(urlencode "${POSTGRES_PASSWORD}") + + cat > /tmp/migration.load < /tmp/migration.load < Date: Sat, 4 Apr 2026 01:37:35 +0200 Subject: [PATCH 19/25] feat(migration): update pgloader script to use placeholders for credentials Co-authored-by: Copilot --- k8s/dev/pgloader-migration.yaml | 19 ++++++++++--------- k8s/prod/pgloader-migration.yaml | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/k8s/dev/pgloader-migration.yaml b/k8s/dev/pgloader-migration.yaml index d9638a23..1c995dbc 100644 --- a/k8s/dev/pgloader-migration.yaml +++ b/k8s/dev/pgloader-migration.yaml @@ -21,27 +21,28 @@ data: MYSQL_PASS_ENCODED=$(urlencode "${MYSQL_PASSWORD}") PG_PASS_ENCODED=$(urlencode "${POSTGRES_PASSWORD}") - cat > /tmp/migration.load < /tmp/migration.load <<'PGEOF' LOAD DATABASE - FROM mysql://root:${MYSQL_PASS_ENCODED}@mysql.udc-bot-dev.svc.cluster.local:3306/udcbot - INTO postgresql://udcbot:${PG_PASS_ENCODED}@postgresql.udc-bot-dev.svc.cluster.local:5432/udcbot + FROM mysql://MYSQL_PLACEHOLDER@mysql.udc-bot-dev.svc.cluster.local:3306/udcbot + INTO postgresql://PG_PLACEHOLDER@postgresql.udc-bot-dev.svc.cluster.local:5432/udcbot WITH data only, - reset sequences, - workers = 2, concurrency = 1 + reset sequences, + workers = 2, concurrency = 1 SET maintenance_work_mem to '128MB' - CAST type int unsigned to integer, - type bigint unsigned to bigint - INCLUDING ONLY TABLE NAMES MATCHING ~/users|casino_users|token_transactions/ BEFORE LOAD DO - \$\$ TRUNCATE users, casino_users, token_transactions CASCADE; \$\$ + $$ TRUNCATE users, casino_users, token_transactions CASCADE; $$ ; PGEOF + # Substitute credentials into the config (avoid shell expansion in heredoc) + sed -i "s|MYSQL_PLACEHOLDER|root:${MYSQL_PASS_ENCODED}|" /tmp/migration.load + sed -i "s|PG_PLACEHOLDER|udcbot:${PG_PASS_ENCODED}|" /tmp/migration.load + echo "=== Starting pgloader migration ===" echo "Source: mysql.udc-bot-dev.svc.cluster.local/udcbot" echo "Target: postgresql.udc-bot-dev.svc.cluster.local/udcbot" diff --git a/k8s/prod/pgloader-migration.yaml b/k8s/prod/pgloader-migration.yaml index 77def280..265d113c 100644 --- a/k8s/prod/pgloader-migration.yaml +++ b/k8s/prod/pgloader-migration.yaml @@ -22,27 +22,28 @@ data: MYSQL_PASS_ENCODED=$(urlencode "${MYSQL_PASSWORD}") PG_PASS_ENCODED=$(urlencode "${POSTGRES_PASSWORD}") - cat > /tmp/migration.load < /tmp/migration.load <<'PGEOF' LOAD DATABASE - FROM mysql://root:${MYSQL_PASS_ENCODED}@mysql.udc-bot-prod.svc.cluster.local:3306/udcbot - INTO postgresql://udcbot:${PG_PASS_ENCODED}@postgresql.udc-bot-prod.svc.cluster.local:5432/udcbot + FROM mysql://MYSQL_PLACEHOLDER@mysql.udc-bot-prod.svc.cluster.local:3306/udcbot + INTO postgresql://PG_PLACEHOLDER@postgresql.udc-bot-prod.svc.cluster.local:5432/udcbot WITH data only, - reset sequences, - workers = 2, concurrency = 1 + reset sequences, + workers = 2, concurrency = 1 SET maintenance_work_mem to '128MB' - CAST type int unsigned to integer, - type bigint unsigned to bigint - INCLUDING ONLY TABLE NAMES MATCHING ~/users|casino_users|token_transactions/ BEFORE LOAD DO - \$\$ TRUNCATE users, casino_users, token_transactions CASCADE; \$\$ + $$ TRUNCATE users, casino_users, token_transactions CASCADE; $$ ; PGEOF + # Substitute credentials into the config (avoid shell expansion in heredoc) + sed -i "s|MYSQL_PLACEHOLDER|root:${MYSQL_PASS_ENCODED}|" /tmp/migration.load + sed -i "s|PG_PLACEHOLDER|udcbot:${PG_PASS_ENCODED}|" /tmp/migration.load + echo "=== Starting pgloader migration (PROD) ===" echo "Source: mysql.udc-bot-prod.svc.cluster.local/udcbot" echo "Target: postgresql.udc-bot-prod.svc.cluster.local/udcbot" From af1c2f5cb4830976dd83f2872978b95e453f4d6c Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Sat, 4 Apr 2026 03:31:42 +0200 Subject: [PATCH 20/25] refactor(casino): align token_transactions PG schema with MySQL to preserve data during migration - Rename TransactionType enum to TransactionKind (avoids DB column name collision) - Replace Type (integer) column with TransactionType (varchar) to match MySQL - Replace DetailsJson (jsonb) column with Description (text) to match MySQL - Add TargetUserID column for migration compatibility - Update CasinoProps constants, repository SQL, and all consumer references --- DiscordBot/Domain/Casino/CasinoUser.cs | 35 ++++++++++++++----- DiscordBot/Extensions/CasinoRepository.cs | 6 ++-- .../Modules/Casino/CasinoSlashModule.cs | 16 ++++----- DiscordBot/Services/Casino/CasinoService.cs | 20 +++++------ DiscordBot/Services/Casino/GameService.cs | 2 +- DiscordBot/Services/DatabaseService.cs | 5 +-- 6 files changed, 52 insertions(+), 32 deletions(-) diff --git a/DiscordBot/Domain/Casino/CasinoUser.cs b/DiscordBot/Domain/Casino/CasinoUser.cs index 9f7e61ab..354321d1 100644 --- a/DiscordBot/Domain/Casino/CasinoUser.cs +++ b/DiscordBot/Domain/Casino/CasinoUser.cs @@ -16,8 +16,17 @@ public class TokenTransaction { public int Id { get; set; } public required string UserID { get; set; } - public long Amount { get; set; } // Can be negative for spending - public TransactionType Type { get; set; } // Enum for transaction types + public string? TargetUserID { get; set; } + public long Amount { get; set; } + public string TransactionType { get; set; } = ""; + + // Computed from TransactionType string — not mapped to DB + [JsonIgnore] + public TransactionKind Kind + { + get => Enum.TryParse(TransactionType, true, out var result) ? result : TransactionKind.Admin; + set => TransactionType = value.ToString(); + } private Dictionary? _details; @@ -28,17 +37,26 @@ public Dictionary? Details set => _details = value; } - // This property will be mapped to the database JSON column - public string? DetailsJson + // Maps to DB column "description" (text). Stores Details dict as JSON, deserializes with fallback for plain text (from MySQL migration) + public string? Description { get => Details != null && Details.Any() ? JsonConvert.SerializeObject(Details) : null; - set => Details = !string.IsNullOrEmpty(value) ? JsonConvert.DeserializeObject>(value) : new Dictionary(); + set + { + if (string.IsNullOrEmpty(value)) + { + _details = new Dictionary(); + return; + } + try { _details = JsonConvert.DeserializeObject>(value); } + catch (JsonException) { _details = new Dictionary { ["text"] = value }; } + } } public DateTime CreatedAt { get; set; } } -public enum TransactionType +public enum TransactionKind { TokenInitialisation, DailyReward, @@ -96,8 +114,9 @@ public static class CasinoProps // TokenTransaction properties public const string TransactionId = nameof(TokenTransaction.Id); public const string TransactionUserID = nameof(TokenTransaction.UserID); + public const string TargetUserID = nameof(TokenTransaction.TargetUserID); public const string Amount = nameof(TokenTransaction.Amount); - public const string TransactionType = nameof(TokenTransaction.Type); - public const string Details = nameof(TokenTransaction.DetailsJson); + public const string TransactionType = nameof(TokenTransaction.TransactionType); + public const string Details = nameof(TokenTransaction.Description); public const string TransactionCreatedAt = nameof(TokenTransaction.CreatedAt); } \ No newline at end of file diff --git a/DiscordBot/Extensions/CasinoRepository.cs b/DiscordBot/Extensions/CasinoRepository.cs index ef81e5fc..6216431e 100644 --- a/DiscordBot/Extensions/CasinoRepository.cs +++ b/DiscordBot/Extensions/CasinoRepository.cs @@ -32,8 +32,8 @@ public interface ICasinoRepo // Token Transaction Operations [Sql($@" - INSERT INTO {CasinoProps.TransactionTableName} ({CasinoProps.TransactionUserID}, {CasinoProps.Amount}, {CasinoProps.TransactionType}, {CasinoProps.Details}, {CasinoProps.TransactionCreatedAt}) - VALUES (@{CasinoProps.TransactionUserID}, @{CasinoProps.Amount}, @{CasinoProps.TransactionType}, @{CasinoProps.Details}::jsonb, @{CasinoProps.TransactionCreatedAt}) + INSERT INTO {CasinoProps.TransactionTableName} ({CasinoProps.TransactionUserID}, {CasinoProps.TargetUserID}, {CasinoProps.Amount}, {CasinoProps.TransactionType}, {CasinoProps.Details}, {CasinoProps.TransactionCreatedAt}) + VALUES (@{CasinoProps.TransactionUserID}, @{CasinoProps.TargetUserID}, @{CasinoProps.Amount}, @{CasinoProps.TransactionType}, @{CasinoProps.Details}, @{CasinoProps.TransactionCreatedAt}) RETURNING *")] Task InsertTransaction(TokenTransaction tokenTransaction); @@ -47,7 +47,7 @@ public interface ICasinoRepo Task ClearAllTransactions(); [Sql($"SELECT * FROM {CasinoProps.TransactionTableName} WHERE {CasinoProps.TransactionType} = @transactionType ORDER BY {CasinoProps.TransactionCreatedAt} DESC")] - Task> GetTransactionsOfType(TransactionType transactionType); + Task> GetTransactionsOfType(string transactionType); // Test connection [Sql($"SELECT COUNT(*) FROM {CasinoProps.CasinoTableName}")] diff --git a/DiscordBot/Modules/Casino/CasinoSlashModule.cs b/DiscordBot/Modules/Casino/CasinoSlashModule.cs index 9d233935..4d30b28e 100644 --- a/DiscordBot/Modules/Casino/CasinoSlashModule.cs +++ b/DiscordBot/Modules/Casino/CasinoSlashModule.cs @@ -380,14 +380,14 @@ public async Task NavigateHistory(string userId, string pageStr, string requestT private (string emoji, string title, string description) FormatTransactionDisplay(TokenTransaction transaction, bool showUserInfo = false) { - var (emoji, title, description) = transaction.Type switch + var (emoji, title, description) = transaction.Kind switch { - TransactionType.TokenInitialisation => ("🎯", "Account Created", ""), - TransactionType.DailyReward => ("📅", "Daily Reward", ""), - TransactionType.Gift => GetGiftDisplay(transaction), - TransactionType.Game => GetGameDisplay(transaction), - TransactionType.Admin => GetAdminDisplay(transaction), - _ => ("❓", transaction.Type.ToString(), "") + TransactionKind.TokenInitialisation => ("🎯", "Account Created", ""), + TransactionKind.DailyReward => ("📅", "Daily Reward", ""), + TransactionKind.Gift => GetGiftDisplay(transaction), + TransactionKind.Game => GetGameDisplay(transaction), + TransactionKind.Admin => GetAdminDisplay(transaction), + _ => ("❓", transaction.TransactionType, "") }; // If showing user info (for all-users view), prepend user name to title @@ -490,7 +490,7 @@ public async Task AddTokens( await Context.Interaction.DeferAsync(ephemeral: true); - await CasinoService.UpdateUserTokens(targetUser.Id.ToString(), amount, TransactionType.Admin, new Dictionary + await CasinoService.UpdateUserTokens(targetUser.Id.ToString(), amount, TransactionKind.Admin, new Dictionary { ["admin"] = Context.User.Id.ToString(), ["action"] = "add" diff --git a/DiscordBot/Services/Casino/CasinoService.cs b/DiscordBot/Services/Casino/CasinoService.cs index 2cecfb2e..653f72a9 100644 --- a/DiscordBot/Services/Casino/CasinoService.cs +++ b/DiscordBot/Services/Casino/CasinoService.cs @@ -39,7 +39,7 @@ public async Task GetOrCreateCasinoUser(string userId) }; var createdUser = await _databaseService.CasinoQuery.InsertCasinoUser(newUser); - await RecordTransaction(userId, _settings.CasinoStartingTokens, TransactionType.TokenInitialisation); + await RecordTransaction(userId, _settings.CasinoStartingTokens, TransactionKind.TokenInitialisation); await _loggingService.LogChannelAndFile($"{ServiceName}: Created new casino user {userId} with {_settings.CasinoStartingTokens} starting tokens"); return createdUser; } @@ -64,11 +64,11 @@ public async Task TransferTokens(string fromUserId, string toUserId, long await _databaseService.CasinoQuery.UpdateTokens(toUserId, toUser.Tokens + amount, DateTime.UtcNow); // Record transactions - await RecordTransaction(fromUserId, -amount, TransactionType.Gift, new Dictionary + await RecordTransaction(fromUserId, -amount, TransactionKind.Gift, new Dictionary { ["to"] = toUserId, }); - await RecordTransaction(toUserId, amount, TransactionType.Gift, new Dictionary + await RecordTransaction(toUserId, amount, TransactionKind.Gift, new Dictionary { ["from"] = fromUserId }); @@ -76,7 +76,7 @@ public async Task TransferTokens(string fromUserId, string toUserId, long return true; } - public async Task UpdateUserTokens(string userId, long deltaTokens, TransactionType transactionType, Dictionary? details = null) + public async Task UpdateUserTokens(string userId, long deltaTokens, TransactionKind transactionType, Dictionary? details = null) { try { @@ -100,7 +100,7 @@ public async Task SetUserTokens(string userId, long amount, string adminUserId) { await _databaseService.CasinoQuery.UpdateTokens(userId, amount, DateTime.UtcNow); - await RecordTransaction(userId, amount, TransactionType.Admin, new Dictionary + await RecordTransaction(userId, amount, TransactionKind.Admin, new Dictionary { ["admin"] = adminUserId, ["action"] = "set" @@ -126,13 +126,13 @@ public async Task> GetAllRecentTransactions(int limit = 1 return transactions.ToList(); } - private async Task RecordTransaction(string userId, long amount, TransactionType type, Dictionary? details = null) + private async Task RecordTransaction(string userId, long amount, TransactionKind type, Dictionary? details = null) { var transaction = new TokenTransaction { UserID = userId, Amount = amount, - Type = type, + Kind = type, CreatedAt = DateTime.UtcNow, Details = details }; @@ -148,7 +148,7 @@ public async Task> GetGameStatistics(IUser user) { try { - var gameTransactions = await _databaseService.CasinoQuery.GetTransactionsOfType(TransactionType.Game); + var gameTransactions = await _databaseService.CasinoQuery.GetTransactionsOfType(nameof(TransactionKind.Game)); // Group transactions by game type var gameGroups = gameTransactions @@ -206,7 +206,7 @@ public async Task GetGameLeaderboard(string? gameName = n { try { - var gameTransactions = await _databaseService.CasinoQuery.GetTransactionsOfType(TransactionType.Game); + var gameTransactions = await _databaseService.CasinoQuery.GetTransactionsOfType(nameof(TransactionKind.Game)); // Filter by game if specified var filteredTransactions = gameTransactions @@ -332,7 +332,7 @@ public bool IsChannelAllowed(ulong channelId) var tokensAwarded = _settings.CasinoDailyRewardTokens; var newBalance = user.Tokens + tokensAwarded; await _databaseService.CasinoQuery.UpdateTokensAndDailyReward(userId, newBalance, now, now); - await RecordTransaction(userId, tokensAwarded, TransactionType.DailyReward); + await RecordTransaction(userId, tokensAwarded, TransactionKind.DailyReward); await _loggingService.LogChannelAndFile($"{ServiceName}: User {userId} claimed daily reward of {tokensAwarded} tokens"); return (true, tokensAwarded, newBalance, now.AddSeconds(_settings.CasinoDailyRewardIntervalSeconds)); diff --git a/DiscordBot/Services/Casino/GameService.cs b/DiscordBot/Services/Casino/GameService.cs index 3f4d8b2c..68dc4fa8 100644 --- a/DiscordBot/Services/Casino/GameService.cs +++ b/DiscordBot/Services/Casino/GameService.cs @@ -87,7 +87,7 @@ public async Task EndGame(IDiscordGameSession session) foreach (var (player, payout) in payouts) { if (player.IsAI) continue; // Skip AI players - await _casinoService.UpdateUserTokens(player.UserId.ToString(), payout, TransactionType.Game, new Dictionary + await _casinoService.UpdateUserTokens(player.UserId.ToString(), payout, TransactionKind.Game, new Dictionary { { "game", session.GameName }, }); diff --git a/DiscordBot/Services/DatabaseService.cs b/DiscordBot/Services/DatabaseService.cs index f073ddf0..5fe2a940 100644 --- a/DiscordBot/Services/DatabaseService.cs +++ b/DiscordBot/Services/DatabaseService.cs @@ -142,9 +142,10 @@ await _logging.LogAction( $"CREATE TABLE {CasinoProps.TransactionTableName} (" + $"{CasinoProps.TransactionId} SERIAL PRIMARY KEY, " + $"{CasinoProps.TransactionUserID} varchar(32) NOT NULL, " + + $"{CasinoProps.TargetUserID} varchar(32) DEFAULT NULL, " + $"{CasinoProps.Amount} bigint NOT NULL, " + - $"{CasinoProps.TransactionType} integer NOT NULL, " + - $"{CasinoProps.Details} jsonb DEFAULT NULL, " + + $"{CasinoProps.TransactionType} varchar(50) NOT NULL, " + + $"{CasinoProps.Details} text DEFAULT NULL, " + $"{CasinoProps.TransactionCreatedAt} timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)"); c.ExecuteSql( From 9e0af1568d36f8ab467d8dbaeaac097804f7f04e Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Sat, 4 Apr 2026 03:32:43 +0200 Subject: [PATCH 21/25] docs: add comprehensive MySQL-to-PostgreSQL code changes reference - Create mysql-to-postgresql-changes.md with full schema, code, and infra details - Update data-migration-mysql-to-postgresql.md with working pgloader config - Delete outdated mysql-to-postgresql-migration.md summary - Update docs/INDEX.md with new doc links --- docs/INDEX.md | 2 + .../data-migration-mysql-to-postgresql.md | 30 +- .../plans/done/mysql-to-postgresql-changes.md | 284 ++++++++++++++++++ .../done/mysql-to-postgresql-migration.md | 49 --- 4 files changed, 308 insertions(+), 57 deletions(-) create mode 100644 docs/plans/done/mysql-to-postgresql-changes.md delete mode 100644 docs/plans/done/mysql-to-postgresql-migration.md diff --git a/docs/INDEX.md b/docs/INDEX.md index 401aef29..75543243 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -40,4 +40,6 @@ post_date: "2026-04-03" | Document | Description | |----------|-------------| +| [MySQL → PostgreSQL Data Migration](plans/data-migration-mysql-to-postgresql.md) | Operational guide — pgloader, cutover phases, kubectl commands | +| [MySQL → PostgreSQL Code Changes](plans/done/mysql-to-postgresql-changes.md) | Architecture reference — all code, schema, and infra changes in PR #375 | | [plans/done/](plans/done/) | Completed feature plans | diff --git a/docs/plans/data-migration-mysql-to-postgresql.md b/docs/plans/data-migration-mysql-to-postgresql.md index e053135f..854abdbc 100644 --- a/docs/plans/data-migration-mysql-to-postgresql.md +++ b/docs/plans/data-migration-mysql-to-postgresql.md @@ -130,27 +130,41 @@ spec: #### Step 2: pgloader Configuration +> **Note:** The actual working config is in `k8s/dev/pgloader-migration.yaml` and `k8s/prod/pgloader-migration.yaml`. See [MySQL → PostgreSQL Code Changes](done/mysql-to-postgresql-changes.md) for all the pgloader quirks discovered during testing. + ``` LOAD DATABASE - FROM mysql://root:PASSWORD@mysql.udc-bot-prod.svc.cluster.local:3306/udcbot - INTO postgresql://udcbot:PASSWORD@postgresql.udc-bot-dev.svc.cluster.local:5432/udcbot + FROM mysql://root:${MYSQL_PASSWORD}@mysql..svc.cluster.local:3306/udcbot + INTO postgresql://udcbot:${POSTGRES_PASSWORD}@postgresql..svc.cluster.local:5432/udcbot -WITH include drop, create tables, create indexes, reset sequences, +WITH data only, reset sequences, workers = 2, concurrency = 1 SET maintenance_work_mem to '128MB' -CAST type int unsigned to integer, - type bigint unsigned to bigint +INCLUDING ONLY TABLE NAMES MATCHING ~/users|casino_users|token_transactions/ --- Only migrate these tables -INCLUDING ONLY TABLE NAMES MATCHING 'users', 'casino_users', 'token_transactions' +ALTER SCHEMA 'udcbot' RENAME TO 'public' BEFORE LOAD DO - $$ TRUNCATE users, casino_users, token_transactions CASCADE; $$ + $$ ALTER TABLE users ADD COLUMN IF NOT EXISTS birthday timestamp; $$, + $$ DROP TABLE IF EXISTS token_transactions CASCADE; $$, + $$ CREATE TABLE token_transactions (Id SERIAL PRIMARY KEY, UserID varchar(32) NOT NULL, TargetUserID varchar(32) DEFAULT NULL, Amount bigint NOT NULL, TransactionType varchar(50) NOT NULL, Description text DEFAULT NULL, CreatedAt timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP); $$, + $$ CREATE INDEX IF NOT EXISTS idx_user_created ON token_transactions (UserID, CreatedAt); $$, + $$ TRUNCATE users, casino_users CASCADE; $$ + +AFTER LOAD DO + $$ ALTER TABLE users DROP COLUMN IF EXISTS birthday; $$ ; ``` +**Key points:** +- `data only` mode — tables must already exist (bot creates them on startup) +- `ALTER SCHEMA 'udcbot' RENAME TO 'public'` — MySQL DB name maps to a PG schema; this redirects to `public` +- Passwords are embedded in URLs — pgloader does **not** respect `PGPASSWORD`/`MYSQL_PWD` env vars +- `token_transactions` is DROP+CREATE'd because the bot's DDL schema may differ from MySQL's column names +- `birthday` column is added temporarily (MySQL has it, PG doesn't) then dropped after import + #### Step 3: Verify ```sql diff --git a/docs/plans/done/mysql-to-postgresql-changes.md b/docs/plans/done/mysql-to-postgresql-changes.md new file mode 100644 index 00000000..e3042459 --- /dev/null +++ b/docs/plans/done/mysql-to-postgresql-changes.md @@ -0,0 +1,284 @@ +# MySQL → PostgreSQL Code & Schema Changes + +> Reference document for PR #375 (`feature/postgre`). Describes **what** was changed in the codebase and **why**. +> +> For the operational data migration procedure, see [data-migration-mysql-to-postgresql.md](../data-migration-mysql-to-postgresql.md). + +## Table of Contents + +- [1. Database Schema Differences](#1-database-schema-differences) +- [2. Application Code Changes](#2-application-code-changes) +- [3. Infrastructure Changes](#3-infrastructure-changes) +- [4. pgloader Gotchas](#4-pgloader-gotchas) +- [5. Files Summary](#5-files-summary) + +--- + +## 1. Database Schema Differences + +### 1.1. Type Mapping + +PostgreSQL does not support unsigned integer types. All `uint`/`ulong` C# properties mapped to DB columns were changed to `int`/`long`. + +| MySQL Type | PostgreSQL Type | C# Type | Affected Columns | +|-----------|----------------|---------|-----------------| +| `int unsigned` | `integer` | `int` | All `Id`, Karma*, KarmaGiven, Level | +| `bigint unsigned` | `bigint` | `long` | Exp, Tokens, Amount | +| `datetime` | `timestamptz` | `DateTime` | CreatedAt, UpdatedAt, LastDailyReward | +| `auto_increment` | `SERIAL` | — | All primary keys | +| `varchar(N)` | `varchar(N)` | `string` | No change | +| `text` | `text` | `string` | No change | + +### 1.2. Identifier Casing + +PostgreSQL lowercases all unquoted identifiers. The DDL in `DatabaseService.cs` uses unquoted names (e.g., `UserID` becomes `userid` in the PG catalog). Insight.Database maps C# PascalCase properties to PG lowercase columns automatically via case-insensitive matching. + +**Never use double-quoted identifiers** in PostgreSQL DDL — it forces case-sensitivity and breaks Insight.Database's automatic mapping. + +### 1.3. `token_transactions` Table + +This table was redesigned to match the MySQL schema exactly, enabling direct data migration via pgloader (which maps columns by name, case-insensitively). + +| Column | MySQL | PostgreSQL (original design) | PostgreSQL (current) | +|--------|-------|------------------------------|---------------------| +| Id | `int unsigned auto_increment` | `SERIAL` | `SERIAL` | +| UserID | `varchar(32) NOT NULL` | `varchar(32) NOT NULL` | `varchar(32) NOT NULL` | +| TargetUserID | `varchar(32) NULL` | *(missing)* | `varchar(32) DEFAULT NULL` | +| Amount | `bigint NOT NULL` | `bigint NOT NULL` | `bigint NOT NULL` | +| TransactionType | `varchar(50) NOT NULL` | `integer NOT NULL` (named `Type`) | `varchar(50) NOT NULL` | +| Description | `text NULL` | `jsonb DEFAULT NULL` (named `DetailsJson`) | `text DEFAULT NULL` | +| CreatedAt | `datetime NOT NULL` | `timestamptz NOT NULL` | `timestamptz NOT NULL` | + +**Why the change:** The original PG design used an integer enum for `Type` and `jsonb` for `DetailsJson`. These column names and types did not match MySQL, so pgloader couldn't map them (it maps by column name). Matching the MySQL schema allows direct, zero-transformation migration. + +**Trade-offs:** +- Lost: `jsonb` native JSON queries and indexing on the Description column +- Gained: Direct migration compatibility, simpler schema, human-readable `TransactionType` values + +### 1.4. `karma_reset_meta` — New Table (PG only) + +MySQL has a built-in EVENT scheduler that resets weekly/monthly/yearly karma columns. **PostgreSQL has no EVENT scheduler.** + +The standard PG alternative is the `pg_cron` extension, but it requires superuser access and isn't always available in managed/containerized deployments. + +**Solution:** A new C# background service (`KarmaResetService`) polls hourly and tracks last-reset timestamps in a `karma_reset_meta` table. This table is created automatically on startup — it has no MySQL counterpart and is not part of the data migration. + +### 1.5. `users.Birthday` Column + +MySQL has a `Birthday` column that was dropped in the PostgreSQL schema. The pgloader migration handles this with temporary column add/drop (see [data-migration-mysql-to-postgresql.md](../data-migration-mysql-to-postgresql.md)). + +--- + +## 2. Application Code Changes + +### 2.1. NuGet Packages + +| Package | Old | New | +|---------|-----|-----| +| `MySql.Data` | (removed) | — | +| `Insight.Database` | 8.0.5 | 8.0.6 | +| `Insight.Database.Providers.PostgreSQL` | (new) | 8.0.6 | + +### 2.2. Database Layer (`DatabaseService.cs`) + +- `MySqlConnection` → `NpgsqlConnection` +- Registered `PostgreSQLInsightDbProvider` on startup +- Full DDL rewrite to PostgreSQL syntax (`SERIAL`, `timestamptz`, no double-quotes, no unsigned types) + +### 2.3. `uint` → `int` / `ulong` → `long` + +Npgsql throws `DbType.UInt32 isn't supported by PostgreSQL or Npgsql` for unsigned C# types. All 17+ files with DB-mapped unsigned properties were changed: + +- `Domain/ProfileData.cs` — karma/level fields +- `Domain/Casino/CasinoUser.cs` — Id +- `Domain/Casino/Game.cs`, `GamePlayer.cs`, `GameSession.cs` +- `Domain/Casino/Games/Cards/Blackjack/Blackjack.cs` +- `Domain/Casino/Games/Cards/Poker/Poker.cs` +- `Domain/Casino/Games/RockPaperScissors/RockPaperScissors.cs` +- `Extensions/CasinoRepository.cs`, `UserDBRepository.cs` +- `Modules/Casino/CasinoSlashModule.cs`, `CasinoSlashModule.Games.cs` +- `Modules/UserModule.cs` +- `Services/Casino/CasinoService.cs`, `GameService.cs` +- `Services/UserService.cs` +- `Settings/Deserialized/Settings.cs` + +### 2.4. SQL Syntax Differences + +| MySQL | PostgreSQL | Files | +|-------|-----------|-------| +| `RAND()` | `RANDOM()` | `UserDBRepository.cs` | +| `INSERT...SELECT LAST_INSERT_ID()` | `INSERT...RETURNING *` | `UserDBRepository.cs`, `CasinoRepository.cs` | +| `SHOW COLUMNS FROM...` | `information_schema.columns` query | `DBConnectionExtension.cs` | +| `::jsonb` cast | Removed (column is now `text`) | `CasinoRepository.cs` | + +### 2.5. `TransactionType` Enum → `TransactionKind` + +Renamed to avoid a naming collision with the new `TransactionType` string DB column property on `TokenTransaction`. + +| Before | After | +|--------|-------| +| `enum TransactionType { ... }` | `enum TransactionKind { ... }` | +| `transaction.Type` (enum property) | `transaction.Kind` (computed from `TransactionType` string) | +| `TransactionType.Game` | `TransactionKind.Game` | + +**How the mapping works:** + +1. DB column `transactiontype` (varchar) maps to `TokenTransaction.TransactionType` (string property) +2. `TokenTransaction.Kind` is a computed property: + - Getter: parses `TransactionType` string to `TransactionKind` enum (case-insensitive, defaults to `Admin`) + - Setter: converts enum to string via `.ToString()` and assigns to `TransactionType` +3. `CasinoProps.TransactionType` constant = `"TransactionType"` (the column name) +4. `GetTransactionsOfType()` now takes a `string` parameter instead of the enum, called with `nameof(TransactionKind.Game)` + +### 2.6. `DetailsJson` → `Description` + +The `DetailsJson` property (mapped to `jsonb`) was replaced by `Description` (mapped to `text`). The `Details` dictionary is still used internally — it's JSON-serialized to plain text. + +**Backward compatibility for migrated MySQL data:** The `Description` setter handles both formats: +- JSON strings (new data from the bot): deserialized to `Dictionary` +- Plain text (old MySQL data): caught via `JsonException`, stored as `{ "text": "" }` + +### 2.7. `TargetUserID` Column + +Added to match MySQL schema. Currently **unused** by application code — the target user for gift transfers is stored in `Details["from"]`/`Details["to"]`. The column exists for migration compatibility and potential future use. + +### 2.8. `KarmaResetService` (new) + +**File:** `Services/KarmaResetService.cs` — registered in `Program.cs` as singleton. + +Replaces MySQL EVENT scheduler: +- Background loop checks hourly +- Resets `KarmaWeekly` on Mondays, `KarmaMonthly` on 1st, `KarmaYearly` on Jan 1st +- Persists timestamps in `karma_reset_meta` table (auto-created) +- Catches up missed resets on startup (e.g., bot was down) + +### 2.9. Other Changes + +- **`DBConnectionExtension.cs`**: Changed `using MySql.Data.MySqlClient` → `using Npgsql`. `ColumnExists` now queries `information_schema.columns` with parameterized SQL and `LOWER()` for case-insensitive matching. +- **`ModerationModule.cs`**: Removed orphaned `BouncyCastle` import. + +--- + +## 3. Infrastructure Changes + +### 3.1. Docker Compose + +| Service | Before | After | +|---------|--------|-------| +| Database | `mysql:8.0` | `postgres:16` | +| Admin UI | phpMyAdmin | `adminer:4` (Dracula theme) | +| Port | 3306 | 5432 | + +Environment variables: `MYSQL_*` → `POSTGRES_*`. Volume: `mysql_data` → `postgres_data`. + +### 3.2. Kubernetes Manifests (dev + prod) + +| File | Change | +|------|--------| +| `postgresql.yaml` | **New.** StatefulSet + Service + PVC. | +| `postgresql-backup.yaml` | **New.** CronJob using `pg_dump` + S3 upload. | +| `mysql-backup.yaml` | **Deleted.** | +| `phpmyadmin.yaml` → `adminer.yaml` | **Renamed and rewritten.** Adminer uses ~13 Mi RAM vs pgAdmin's ~194 Mi. | +| `pgadmin.yaml` | **Deleted** (was added then replaced by Adminer). | +| `pgloader-migration.yaml` | **New.** ConfigMap + Job for data migration. | +| `external-secrets.yaml` | Added `postgresql-credentials`. MySQL secrets kept temporarily. | +| `bot-config.yaml` | Connection string changed to PostgreSQL format. | +| `bot.yaml` | Init container waits for port 5432 instead of 3306. | + +### 3.3. ExternalSecrets (1Password) + +**New permanent items:** + +| 1Password Item | K8s Secret | Purpose | +|----------------|-----------|---------| +| `PostgreSQL Server - Dev` | `postgresql-credentials` | PG password (dev) | +| `PostgreSQL Server - Prod` | `postgresql-credentials` | PG password (prod) | + +**Temporary items (remove after migration):** + +| 1Password Item | K8s Secret | Purpose | +|----------------|-----------|---------| +| `MySQL Server - Root User - Dev/Prod` | `mysql-credentials` | For pgloader | +| `MySQL Server - UDC User - Dev/Prod` | `mysql-user-credentials` | For pgloader | + +### 3.4. Resource Comparison + +Dev PostgreSQL stack uses ~47% less RAM than prod MySQL stack: + +| Component | MySQL Stack (prod) | PostgreSQL Stack (dev) | +|-----------|-------------------|----------------------| +| Database | ~614 Mi | ~303 Mi | +| Admin UI | ~230 Mi (phpMyAdmin) | ~13 Mi (Adminer) | +| **Total** | **~844 Mi** | **~400 Mi (est.)** | + +--- + +## 4. pgloader Gotchas + +Lessons learned from 5 iterations of pgloader testing: + +| Issue | What Happened | Solution | +|-------|--------------|---------| +| `CAST` not supported in `data only` mode | pgloader threw syntax error on `CAST type int unsigned to integer` | Remove CAST — pgloader handles type coercion automatically in data-only mode | +| Env vars ignored | `PGPASSWORD` and `MYSQL_PWD` had no effect | Embed passwords directly in connection URLs (pgloader uses its own connection libraries) | +| Schema mismatch | pgloader couldn't find schema "udcbot" in PostgreSQL | Add `ALTER SCHEMA 'udcbot' RENAME TO 'public'` — MySQL DB name maps to a PG schema | +| Column name mismatch | `token_transactions` had different column names in MySQL vs PG | Changed PG schema to match MySQL column names (see section 1.3) | +| Missing `Birthday` column | MySQL `users` has `Birthday`, PG doesn't | Temp `ALTER TABLE ADD/DROP COLUMN` in BEFORE/AFTER LOAD | + +--- + +## 5. Files Summary + +### New Files + +| File | Purpose | +|------|---------| +| `Services/KarmaResetService.cs` | Replaces MySQL EVENT scheduler | +| `k8s/*/postgresql.yaml` | PostgreSQL StatefulSet + Service + PVC | +| `k8s/*/postgresql-backup.yaml` | CronJob for PG backups to S3 | +| `k8s/*/adminer.yaml` | Adminer web UI | +| `k8s/*/pgloader-migration.yaml` | Data migration Job | +| `docs/plans/data-migration-mysql-to-postgresql.md` | Operational migration guide | +| `docs/plans/mysql-to-postgresql-changes.md` | This document | +| `docs/plans/done/mysql-to-postgresql-migration.md` | Original PR summary | + +### Modified Files (17+ C# + infra) + +| File | Change | +|------|--------| +| `DiscordBot.csproj` | NuGet changes | +| `Domain/Casino/CasinoUser.cs` | `uint`→`int`, enum rename, schema property renames | +| `Domain/ProfileData.cs` | `uint`→`int` | +| `Domain/Casino/Game*.cs` (4 files) | `uint`→`int` | +| `Extensions/CasinoRepository.cs` | SQL syntax, parameter types | +| `Extensions/DBConnectionExtension.cs` | MySQL→Npgsql, `information_schema` | +| `Extensions/UserDBRepository.cs` | `RAND()`→`RANDOM()`, `RETURNING *` | +| `Modules/Casino/CasinoSlashModule*.cs` | Enum rename | +| `Modules/ModerationModule.cs` | Removed orphaned import | +| `Modules/UserModule.cs` | `uint`→`int` | +| `Services/DatabaseService.cs` | Complete DDL rewrite | +| `Services/Casino/CasinoService.cs` | Enum rename | +| `Services/Casino/GameService.cs` | Enum rename | +| `Services/UserService.cs` | `uint`→`int`, path changes | +| `Settings/Deserialized/Settings.cs` | `uint`→`int` | +| `Program.cs` | Registered `KarmaResetService` | +| `docker-compose.yml` | MySQL→PostgreSQL, phpMyAdmin→Adminer | +| `README.md` | Updated setup instructions | +| `k8s/*/external-secrets.yaml` | Added PG secrets | +| `k8s/*/bot-config.yaml` | PG connection string | +| `k8s/*/bot.yaml` | Init container port 5432 | + +### Deleted Files + +| File | Reason | +|------|--------| +| `k8s/*/mysql-backup.yaml` | Replaced by `postgresql-backup.yaml` | +| `k8s/*/pgadmin.yaml` | Replaced by Adminer | + +### Temporary Files (remove after migration) + +| File | Remove When | +|------|------------| +| `k8s/*/mysql.yaml` | After prod data migration verified (24-48h) | +| `k8s/*/pgloader-migration.yaml` | After prod migration complete | +| MySQL ExternalSecret entries | After MySQL fully decommissioned | diff --git a/docs/plans/done/mysql-to-postgresql-migration.md b/docs/plans/done/mysql-to-postgresql-migration.md deleted file mode 100644 index f1b8e7ab..00000000 --- a/docs/plans/done/mysql-to-postgresql-migration.md +++ /dev/null @@ -1,49 +0,0 @@ -# MySQL → PostgreSQL Migration - -**Status:** Done -**Date:** 2025-07-15 - -## Summary - -Migrated the entire data layer and infrastructure from MySQL 8.0 to PostgreSQL 16. - -## Changes - -### Application Code - -- **NuGet:** Removed `MySql.Data`, added `Insight.Database.Providers.PostgreSQL` (brings Npgsql transitively) -- **DatabaseService:** `MySqlConnection` → `NpgsqlConnection`, DDL rewritten to PostgreSQL syntax, registered `PostgreSQLInsightDbProvider` -- **DBConnectionExtension:** `SHOW COLUMNS` → `information_schema` query -- **UserDBRepository:** `RAND()` → `RANDOM()`, `INSERT...SELECT` → `INSERT...RETURNING *` -- **CasinoRepository:** `LAST_INSERT_ID()` → `RETURNING *`, added `::jsonb` cast -- **KarmaResetService (new):** Replaces MySQL EVENT scheduler with C# polling loop for weekly/monthly/yearly karma resets -- **Program.cs:** Registered `KarmaResetService` -- **ModerationModule:** Removed orphaned `BouncyCastle` import - -### Infrastructure - -- **docker-compose.yml:** MySQL → postgres:16, phpMyAdmin → pgAdmin4 -- **K8s manifests (dev + prod):** Renamed and rewrote mysql.yaml → postgresql.yaml, mysql-backup.yaml → postgresql-backup.yaml, phpmyadmin.yaml → pgadmin.yaml -- **K8s bot.yaml:** Init container updated to wait for PostgreSQL on port 5432 -- **K8s external-secrets.yaml:** MySQL secrets → `postgresql-credentials` + `pgadmin-credentials` -- **K8s bot-config.yaml:** PostgreSQL connection string format -- **Settings.example.json:** Updated connection string and removed XAMPP comment - -### Documentation - -- **README.md:** Updated manual database setup instructions for PostgreSQL - -## Checklist - -- [x] NuGet packages updated -- [x] All SQL queries ported to PostgreSQL syntax -- [x] MySQL EVENT scheduler replaced with KarmaResetService -- [x] Docker Compose updated -- [x] K8s manifests updated and renamed (dev + prod) -- [x] Settings and connection strings updated -- [x] Build verified (0 errors) -- [x] No remaining MySQL references (except historical comment in KarmaResetService) - -## Data Migration Note - -Existing MySQL data needs to be manually exported and imported into PostgreSQL using `pg_dump`/`pg_restore` or a migration tool like `pgloader`. From d6223c0f97842fa62b273de54bfbc8e8d787da92 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Sat, 4 Apr 2026 03:33:12 +0200 Subject: [PATCH 22/25] feat(k8s): include token_transactions in pgloader migration configs - Add token_transactions to table matching regex (dev + prod) - Add BEFORE LOAD: DROP+CREATE token_transactions with MySQL-compatible schema - Ensures existing PG tables with old column names are replaced before import --- k8s/dev/pgloader-migration.yaml | 29 +++++++++++++---------------- k8s/prod/pgloader-migration.yaml | 29 +++++++++++++---------------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/k8s/dev/pgloader-migration.yaml b/k8s/dev/pgloader-migration.yaml index 1c995dbc..b65092f8 100644 --- a/k8s/dev/pgloader-migration.yaml +++ b/k8s/dev/pgloader-migration.yaml @@ -13,18 +13,10 @@ data: #!/bin/sh set -e - # URL-encode passwords using pure shell (handles special chars like %) - urlencode() { - printf '%s' "$1" | od -An -tx1 | tr ' ' '%' | tr -d '\n' | sed 's/^%//' - } - - MYSQL_PASS_ENCODED=$(urlencode "${MYSQL_PASSWORD}") - PG_PASS_ENCODED=$(urlencode "${POSTGRES_PASSWORD}") - - cat > /tmp/migration.load <<'PGEOF' + cat > /tmp/migration.load < /tmp/migration.load <<'PGEOF' + cat > /tmp/migration.load < Date: Sat, 4 Apr 2026 03:37:17 +0200 Subject: [PATCH 23/25] chore(k8s): update bot image to version d6223c0 --- k8s/dev/bot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/dev/bot.yaml b/k8s/dev/bot.yaml index 4f45e86e..5754624f 100644 --- a/k8s/dev/bot.yaml +++ b/k8s/dev/bot.yaml @@ -128,7 +128,7 @@ spec: - ALL containers: - name: bot - image: ghcr.io/unity-developer-community/udc-bot-dev:17fc156 + image: ghcr.io/unity-developer-community/udc-bot-dev:d6223c0 volumeMounts: - name: app-settings mountPath: /app/Settings From 46ede2c6bddcdd8d50d75f3058be95a7bb57d0b3 Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Sat, 4 Apr 2026 05:28:25 +0200 Subject: [PATCH 24/25] rework doc Co-authored-by: Copilot --- .../plans/done/mysql-to-postgresql-changes.md | 58 +++++++------------ 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/docs/plans/done/mysql-to-postgresql-changes.md b/docs/plans/done/mysql-to-postgresql-changes.md index e3042459..86c35b53 100644 --- a/docs/plans/done/mysql-to-postgresql-changes.md +++ b/docs/plans/done/mysql-to-postgresql-changes.md @@ -37,23 +37,19 @@ PostgreSQL lowercases all unquoted identifiers. The DDL in `DatabaseService.cs` ### 1.3. `token_transactions` Table -This table was redesigned to match the MySQL schema exactly, enabling direct data migration via pgloader (which maps columns by name, case-insensitively). +The PostgreSQL schema matches the MySQL schema to allow direct data migration via pgloader (which maps columns by name, case-insensitively). -| Column | MySQL | PostgreSQL (original design) | PostgreSQL (current) | -|--------|-------|------------------------------|---------------------| -| Id | `int unsigned auto_increment` | `SERIAL` | `SERIAL` | -| UserID | `varchar(32) NOT NULL` | `varchar(32) NOT NULL` | `varchar(32) NOT NULL` | -| TargetUserID | `varchar(32) NULL` | *(missing)* | `varchar(32) DEFAULT NULL` | -| Amount | `bigint NOT NULL` | `bigint NOT NULL` | `bigint NOT NULL` | -| TransactionType | `varchar(50) NOT NULL` | `integer NOT NULL` (named `Type`) | `varchar(50) NOT NULL` | -| Description | `text NULL` | `jsonb DEFAULT NULL` (named `DetailsJson`) | `text DEFAULT NULL` | -| CreatedAt | `datetime NOT NULL` | `timestamptz NOT NULL` | `timestamptz NOT NULL` | +| Column | MySQL | PostgreSQL | +|--------|-------|-----------| +| Id | `int unsigned auto_increment` | `SERIAL` | +| UserID | `varchar(32) NOT NULL` | `varchar(32) NOT NULL` | +| TargetUserID | `varchar(32) NULL` | `varchar(32) DEFAULT NULL` | +| Amount | `bigint NOT NULL` | `bigint NOT NULL` | +| TransactionType | `varchar(50) NOT NULL` | `varchar(50) NOT NULL` | +| Description | `text NULL` | `text DEFAULT NULL` | +| CreatedAt | `datetime NOT NULL` | `timestamptz NOT NULL` | -**Why the change:** The original PG design used an integer enum for `Type` and `jsonb` for `DetailsJson`. These column names and types did not match MySQL, so pgloader couldn't map them (it maps by column name). Matching the MySQL schema allows direct, zero-transformation migration. - -**Trade-offs:** -- Lost: `jsonb` native JSON queries and indexing on the Description column -- Gained: Direct migration compatibility, simpler schema, human-readable `TransactionType` values +**Design note:** `TransactionType` is stored as a human-readable string (e.g., `"Game"`, `"Gift"`) rather than an integer enum. `Description` stores JSON-serialized metadata as plain text rather than using PostgreSQL's `jsonb` type. Both choices preserve compatibility with the existing MySQL data. ### 1.4. `karma_reset_meta` — New Table (PG only) @@ -63,10 +59,6 @@ The standard PG alternative is the `pg_cron` extension, but it requires superuse **Solution:** A new C# background service (`KarmaResetService`) polls hourly and tracks last-reset timestamps in a `karma_reset_meta` table. This table is created automatically on startup — it has no MySQL counterpart and is not part of the data migration. -### 1.5. `users.Birthday` Column - -MySQL has a `Birthday` column that was dropped in the PostgreSQL schema. The pgloader migration handles this with temporary column add/drop (see [data-migration-mysql-to-postgresql.md](../data-migration-mysql-to-postgresql.md)). - --- ## 2. Application Code Changes @@ -111,15 +103,9 @@ Npgsql throws `DbType.UInt32 isn't supported by PostgreSQL or Npgsql` for unsign | `SHOW COLUMNS FROM...` | `information_schema.columns` query | `DBConnectionExtension.cs` | | `::jsonb` cast | Removed (column is now `text`) | `CasinoRepository.cs` | -### 2.5. `TransactionType` Enum → `TransactionKind` - -Renamed to avoid a naming collision with the new `TransactionType` string DB column property on `TokenTransaction`. +### 2.5. `TransactionKind` Enum and `TransactionType` Column -| Before | After | -|--------|-------| -| `enum TransactionType { ... }` | `enum TransactionKind { ... }` | -| `transaction.Type` (enum property) | `transaction.Kind` (computed from `TransactionType` string) | -| `TransactionType.Game` | `TransactionKind.Game` | +The C# enum is called `TransactionKind` (not `TransactionType`) to avoid a naming collision with the `TransactionType` string property on `TokenTransaction` that maps to the DB `transactiontype` varchar(50) column. **How the mapping works:** @@ -128,15 +114,15 @@ Renamed to avoid a naming collision with the new `TransactionType` string DB col - Getter: parses `TransactionType` string to `TransactionKind` enum (case-insensitive, defaults to `Admin`) - Setter: converts enum to string via `.ToString()` and assigns to `TransactionType` 3. `CasinoProps.TransactionType` constant = `"TransactionType"` (the column name) -4. `GetTransactionsOfType()` now takes a `string` parameter instead of the enum, called with `nameof(TransactionKind.Game)` +4. `GetTransactionsOfType()` takes a `string` parameter, called with `nameof(TransactionKind.Game)` -### 2.6. `DetailsJson` → `Description` +### 2.6. `Description` Column and `Details` Dictionary -The `DetailsJson` property (mapped to `jsonb`) was replaced by `Description` (mapped to `text`). The `Details` dictionary is still used internally — it's JSON-serialized to plain text. +The DB column `description` (text) stores JSON-serialized metadata as plain text. In C#, `TokenTransaction.Description` is the DB-mapped property, while `TokenTransaction.Details` (Dictionary) is the in-memory representation. -**Backward compatibility for migrated MySQL data:** The `Description` setter handles both formats: -- JSON strings (new data from the bot): deserialized to `Dictionary` -- Plain text (old MySQL data): caught via `JsonException`, stored as `{ "text": "" }` +The `Description` setter handles both formats: +- JSON strings (normal bot data): deserialized to `Dictionary` +- Plain text (migrated MySQL data): caught via `JsonException`, stored as `{ "text": "" }` ### 2.7. `TargetUserID` Column @@ -179,7 +165,7 @@ Environment variables: `MYSQL_*` → `POSTGRES_*`. Volume: `mysql_data` → `pos | `postgresql-backup.yaml` | **New.** CronJob using `pg_dump` + S3 upload. | | `mysql-backup.yaml` | **Deleted.** | | `phpmyadmin.yaml` → `adminer.yaml` | **Renamed and rewritten.** Adminer uses ~13 Mi RAM vs pgAdmin's ~194 Mi. | -| `pgadmin.yaml` | **Deleted** (was added then replaced by Adminer). | +| `pgadmin.yaml` | **Deleted.** | | `pgloader-migration.yaml` | **New.** ConfigMap + Job for data migration. | | `external-secrets.yaml` | Added `postgresql-credentials`. MySQL secrets kept temporarily. | | `bot-config.yaml` | Connection string changed to PostgreSQL format. | @@ -223,7 +209,6 @@ Lessons learned from 5 iterations of pgloader testing: | Env vars ignored | `PGPASSWORD` and `MYSQL_PWD` had no effect | Embed passwords directly in connection URLs (pgloader uses its own connection libraries) | | Schema mismatch | pgloader couldn't find schema "udcbot" in PostgreSQL | Add `ALTER SCHEMA 'udcbot' RENAME TO 'public'` — MySQL DB name maps to a PG schema | | Column name mismatch | `token_transactions` had different column names in MySQL vs PG | Changed PG schema to match MySQL column names (see section 1.3) | -| Missing `Birthday` column | MySQL `users` has `Birthday`, PG doesn't | Temp `ALTER TABLE ADD/DROP COLUMN` in BEFORE/AFTER LOAD | --- @@ -239,8 +224,7 @@ Lessons learned from 5 iterations of pgloader testing: | `k8s/*/adminer.yaml` | Adminer web UI | | `k8s/*/pgloader-migration.yaml` | Data migration Job | | `docs/plans/data-migration-mysql-to-postgresql.md` | Operational migration guide | -| `docs/plans/mysql-to-postgresql-changes.md` | This document | -| `docs/plans/done/mysql-to-postgresql-migration.md` | Original PR summary | +| `docs/plans/done/mysql-to-postgresql-changes.md` | This document | ### Modified Files (17+ C# + infra) From 9203c79e618d9dcfd029fee77f51b1cd087d0f4f Mon Sep 17 00:00:00 2001 From: Pierre Demessence Date: Sat, 4 Apr 2026 05:28:34 +0200 Subject: [PATCH 25/25] fix(docs): clarify handling of birthday column in migration Co-authored-by: Copilot --- docs/plans/data-migration-mysql-to-postgresql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/data-migration-mysql-to-postgresql.md b/docs/plans/data-migration-mysql-to-postgresql.md index 854abdbc..040a0a1f 100644 --- a/docs/plans/data-migration-mysql-to-postgresql.md +++ b/docs/plans/data-migration-mysql-to-postgresql.md @@ -163,7 +163,7 @@ AFTER LOAD DO - `ALTER SCHEMA 'udcbot' RENAME TO 'public'` — MySQL DB name maps to a PG schema; this redirects to `public` - Passwords are embedded in URLs — pgloader does **not** respect `PGPASSWORD`/`MYSQL_PWD` env vars - `token_transactions` is DROP+CREATE'd because the bot's DDL schema may differ from MySQL's column names -- `birthday` column is added temporarily (MySQL has it, PG doesn't) then dropped after import +- `birthday` temp column: added before load and dropped after — handles MySQL sources that may have this column (idempotent, safe for sources without it) #### Step 3: Verify