From a7ad7e28447f7ab3d692fb0e8e45d6a7f39a3cd2 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Sun, 12 Oct 2025 23:15:07 +0900 Subject: [PATCH 01/11] rails g migration add_html_body_to_messages html_body:text --- db/migrate/20251012141446_add_html_body_to_messages.rb | 5 +++++ db/schema.rb | 1 + 2 files changed, 6 insertions(+) create mode 100644 db/migrate/20251012141446_add_html_body_to_messages.rb diff --git a/db/migrate/20251012141446_add_html_body_to_messages.rb b/db/migrate/20251012141446_add_html_body_to_messages.rb new file mode 100644 index 0000000..2c10b6e --- /dev/null +++ b/db/migrate/20251012141446_add_html_body_to_messages.rb @@ -0,0 +1,5 @@ +class AddHtmlBodyToMessages < ActiveRecord::Migration[8.0] + def change + add_column :messages, :html_body, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 9b743e7..c35cd11 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -26,6 +26,7 @@ t.timestamptz "published_at" t.string "message_id_header" t.integer "parent_id" + t.text "html_body" t.index ["body"], name: "index_messages_on_body", opclass: :gin_trgm_ops, using: :gin t.index ["list_id", "list_seq"], name: "index_messages_on_list_id_and_list_seq", unique: true end From f9b64fbc23777d7acace8d93ef42cc2154e9b0a7 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 21 Oct 2025 23:09:37 +0900 Subject: [PATCH 02/11] Message.from_mail is now an instance method --- app/models/message.rb | 72 ++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/app/models/message.rb b/app/models/message.rb index bd4f2c6..ee497de 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -14,44 +14,54 @@ class Message < ApplicationRecord class << self def from_mail(mail, list, list_seq) - body = Kconv.toutf8 mail.body.raw_source - if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316])) - body.gsub!("\u0000", '') - end - if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106]) - mail.header[:subject].value.chop! - end - if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710])) - mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value - end - subject = mail.subject - subject = Kconv.toutf8 subject if subject - from = Kconv.toutf8 mail.from_address&.raw - if !from && (list.name == 'ruby-core') && (list_seq == 161) - from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R - end + new.from_mail(mail, list, list_seq) + end + end - message_id = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace + def from_mail(mail, list, list_seq) + self.list_id, self.list_seq, self.published_at = list.id, list_seq, mail.date - # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value - parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value) - parent_message_id = Message.where(list_id: list.id, message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header - if !parent_message_id && (String === mail.references) - parent_message_id = Message.where(list_id: list.id, message_id_header: mail.references).pick(:id) - end - if !parent_message_id && (Array === mail.references) - mail.references.compact.each do |ref| - break if (parent_message_id = Message.where(list_id: list.id, message_id_header: ref).pick(:id)) - end - end + self.body = Kconv.toutf8 mail.body.raw_source + if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316])) + self.body.gsub!("\u0000", '') + end + + if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106]) + mail.header[:subject].value.chop! + end + if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710])) + mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value + end + self.subject = mail.subject + self.subject = Kconv.toutf8 subject if self.subject - new list_id: list.id, list_seq: list_seq, body: body, subject: subject, from: from, published_at: mail.date, message_id_header: message_id, parent_id: parent_message_id + self.from = Kconv.toutf8 mail.from_address&.raw + if !self.from && (list.name == 'ruby-core') && (list_seq == 161) + self.from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R end - private def extract_message_id_from_in_reply_to(header) - header && header.strip.scan(/<([^>]+)>/).flatten.first + self.message_id_header = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace + + # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value + parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value) + self.parent_id = Message.where(list_id: list.id, message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header + if !self.parent_id && (String === mail.references) + self.parent_id = Message.where(list_id: list.id, message_id_header: mail.references).pick(:id) + end + if !self.parent_id && (Array === mail.references) + mail.references.compact.each do |ref| + break if (self.parent_id = Message.where(list_id: list.id, message_id_header: ref).pick(:id)) + end end + self + end + + private def extract_message_id_from_in_reply_to(header) + header && header.strip.scan(/<([^>]+)>/).flatten.first + end + + class << self def from_s3(list_name, list_seq, s3_client = Aws::S3::Client.new(region: BLADE_BUCKET_REGION)) obj = s3_client.get_object(bucket: BLADE_BUCKET_NAME, key: "#{list_name}/#{list_seq}") From 137b92f0b29379ad67de96a2978ea4110ab58559 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 21 Oct 2025 23:28:23 +0900 Subject: [PATCH 03/11] Save text/html body to html_body column --- app/models/message.rb | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/app/models/message.rb b/app/models/message.rb index ee497de..7480ebf 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -21,7 +21,14 @@ def from_mail(mail, list, list_seq) def from_mail(mail, list, list_seq) self.list_id, self.list_seq, self.published_at = list.id, list_seq, mail.date - self.body = Kconv.toutf8 mail.body.raw_source + if mail.multipart? + mail.parts.each do |p| + handle_multipart p + end + else + self.body = Kconv.toutf8 mail.body.raw_source + end + if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316])) self.body.gsub!("\u0000", '') end @@ -57,6 +64,17 @@ def from_mail(mail, list, list_seq) self end + private def handle_multipart(part) + case part.content_type.downcase + when /^text\/plain/ + (self.body ||= '') << Kconv.toutf8(part.body.raw_source) + when /^text\/html;/ + (self.html_body ||= '') << Kconv.toutf8(part.body.raw_source) + else + puts "Unknown content_type: #{part.content_type}" + end + end + private def extract_message_id_from_in_reply_to(header) header && header.strip.scan(/<([^>]+)>/).flatten.first end From c3570a6dabe736e6d78338e3631ec94d1ba37e1f Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 14 Oct 2025 23:07:00 +0900 Subject: [PATCH 04/11] rails g model active_storage_file_blob key:string:uniq data:binary --- app/models/active_storage_file_blob.rb | 2 ++ ...20251014140646_create_active_storage_file_blobs.rb | 11 +++++++++++ db/schema.rb | 8 ++++++++ test/fixtures/active_storage_file_blobs.yml | 9 +++++++++ test/models/active_storage_file_blob_test.rb | 7 +++++++ 5 files changed, 37 insertions(+) create mode 100644 app/models/active_storage_file_blob.rb create mode 100644 db/migrate/20251014140646_create_active_storage_file_blobs.rb create mode 100644 test/fixtures/active_storage_file_blobs.yml create mode 100644 test/models/active_storage_file_blob_test.rb diff --git a/app/models/active_storage_file_blob.rb b/app/models/active_storage_file_blob.rb new file mode 100644 index 0000000..ecee561 --- /dev/null +++ b/app/models/active_storage_file_blob.rb @@ -0,0 +1,2 @@ +class ActiveStorageFileBlob < ApplicationRecord +end diff --git a/db/migrate/20251014140646_create_active_storage_file_blobs.rb b/db/migrate/20251014140646_create_active_storage_file_blobs.rb new file mode 100644 index 0000000..0b9bfbc --- /dev/null +++ b/db/migrate/20251014140646_create_active_storage_file_blobs.rb @@ -0,0 +1,11 @@ +class CreateActiveStorageFileBlobs < ActiveRecord::Migration[8.0] + def change + create_table :active_storage_file_blobs do |t| + t.string :key + t.binary :data + + t.timestamps + end + add_index :active_storage_file_blobs, :key, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index c35cd11..3081222 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -15,6 +15,14 @@ enable_extension "pg_catalog.plpgsql" enable_extension "pg_trgm" + create_table "active_storage_file_blobs", force: :cascade do |t| + t.string "key" + t.binary "data" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["key"], name: "index_active_storage_file_blobs_on_key", unique: true + end + create_table "messages", force: :cascade do |t| t.string "subject" t.string "from" diff --git a/test/fixtures/active_storage_file_blobs.yml b/test/fixtures/active_storage_file_blobs.yml new file mode 100644 index 0000000..8d0bcc7 --- /dev/null +++ b/test/fixtures/active_storage_file_blobs.yml @@ -0,0 +1,9 @@ +# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +one: + key: MyString + data: + +two: + key: MyString + data: diff --git a/test/models/active_storage_file_blob_test.rb b/test/models/active_storage_file_blob_test.rb new file mode 100644 index 0000000..038dc32 --- /dev/null +++ b/test/models/active_storage_file_blob_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class ActiveStorageFileBlobTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 4d9473cf01a0939aa8ff7690089dc0090526a6ab Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Wed, 22 Oct 2025 01:06:44 +0900 Subject: [PATCH 05/11] gomi --- test/fixtures/active_storage_file_blobs.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 test/fixtures/active_storage_file_blobs.yml diff --git a/test/fixtures/active_storage_file_blobs.yml b/test/fixtures/active_storage_file_blobs.yml deleted file mode 100644 index 8d0bcc7..0000000 --- a/test/fixtures/active_storage_file_blobs.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html - -one: - key: MyString - data: - -two: - key: MyString - data: From b1eb8484489d3513e0d2d9b8490cf2d843254ed5 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 14 Oct 2025 23:11:51 +0900 Subject: [PATCH 06/11] Following the convention --- app/models/active_storage/file_blob.rb | 2 ++ app/models/active_storage_file_blob.rb | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 app/models/active_storage/file_blob.rb delete mode 100644 app/models/active_storage_file_blob.rb diff --git a/app/models/active_storage/file_blob.rb b/app/models/active_storage/file_blob.rb new file mode 100644 index 0000000..b5ba979 --- /dev/null +++ b/app/models/active_storage/file_blob.rb @@ -0,0 +1,2 @@ +class ActiveStorage::FileBlob < ApplicationRecord +end diff --git a/app/models/active_storage_file_blob.rb b/app/models/active_storage_file_blob.rb deleted file mode 100644 index ecee561..0000000 --- a/app/models/active_storage_file_blob.rb +++ /dev/null @@ -1,2 +0,0 @@ -class ActiveStorageFileBlob < ApplicationRecord -end From d4d7f8175c46c675dcb44feb524e04ad3240efae Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 14 Oct 2025 23:13:35 +0900 Subject: [PATCH 07/11] An Active Storage service that stores the file binary in RDB --- .../service/database_service.rb | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 lib/active_storage/service/database_service.rb diff --git a/lib/active_storage/service/database_service.rb b/lib/active_storage/service/database_service.rb new file mode 100644 index 0000000..6dfd866 --- /dev/null +++ b/lib/active_storage/service/database_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'active_storage/service' + +module ActiveStorage + class Service::DatabaseService < Service + def upload(key, io, checksum: nil, **) + instrument :upload, key: key, checksum: checksum do + ActiveStorage::FileBlob.find_or_initialize_by(key: key) do + it.data = io.read + it.save! + end + end + end + + def download(key) + instrument :download, key: key do + ActiveStorage::FileBlob.where(key: key).pick(:data) + end + end + + def delete(key) + instrument :delete, key: key do + ActiveStorage::FileBlob.find_by(key: key)&.destroy + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + payload[:exist] = ActiveStorage::FileBlob.exists?(key: key) + end + end + + private + + def service_name + 'Database' + end + end +end From f133cc57f4c27debcd70de7a3a8bc8f5d013efad Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 14 Oct 2025 23:18:37 +0900 Subject: [PATCH 08/11] bin/rails active_storage:install --- ...te_active_storage_tables.active_storage.rb | 57 +++++++++++++++++++ db/schema.rb | 31 ++++++++++ 2 files changed, 88 insertions(+) create mode 100644 db/migrate/20251014141822_create_active_storage_tables.active_storage.rb diff --git a/db/migrate/20251014141822_create_active_storage_tables.active_storage.rb b/db/migrate/20251014141822_create_active_storage_tables.active_storage.rb new file mode 100644 index 0000000..6bd8bd0 --- /dev/null +++ b/db/migrate/20251014141822_create_active_storage_tables.active_storage.rb @@ -0,0 +1,57 @@ +# This migration comes from active_storage (originally 20170806125915) +class CreateActiveStorageTables < ActiveRecord::Migration[7.0] + def change + # Use Active Record's configured type for primary and foreign keys + primary_key_type, foreign_key_type = primary_and_foreign_key_types + + create_table :active_storage_blobs, id: primary_key_type do |t| + t.string :key, null: false + t.string :filename, null: false + t.string :content_type + t.text :metadata + t.string :service_name, null: false + t.bigint :byte_size, null: false + t.string :checksum + + if connection.supports_datetime_with_precision? + t.datetime :created_at, precision: 6, null: false + else + t.datetime :created_at, null: false + end + + t.index [ :key ], unique: true + end + + create_table :active_storage_attachments, id: primary_key_type do |t| + t.string :name, null: false + t.references :record, null: false, polymorphic: true, index: false, type: foreign_key_type + t.references :blob, null: false, type: foreign_key_type + + if connection.supports_datetime_with_precision? + t.datetime :created_at, precision: 6, null: false + else + t.datetime :created_at, null: false + end + + t.index [ :record_type, :record_id, :name, :blob_id ], name: :index_active_storage_attachments_uniqueness, unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + + create_table :active_storage_variant_records, id: primary_key_type do |t| + t.belongs_to :blob, null: false, index: false, type: foreign_key_type + t.string :variation_digest, null: false + + t.index [ :blob_id, :variation_digest ], name: :index_active_storage_variant_records_uniqueness, unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + end + + private + def primary_and_foreign_key_types + config = Rails.configuration.generators + setting = config.options[config.orm][:primary_key_type] + primary_key_type = setting || :primary_key + foreign_key_type = setting || :bigint + [ primary_key_type, foreign_key_type ] + end +end diff --git a/db/schema.rb b/db/schema.rb index 3081222..43477c3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -15,6 +15,28 @@ enable_extension "pg_catalog.plpgsql" enable_extension "pg_trgm" + create_table "active_storage_attachments", force: :cascade do |t| + t.string "name", null: false + t.string "record_type", null: false + t.bigint "record_id", null: false + t.bigint "blob_id", null: false + t.datetime "created_at", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + end + + create_table "active_storage_blobs", force: :cascade do |t| + t.string "key", null: false + t.string "filename", null: false + t.string "content_type" + t.text "metadata" + t.string "service_name", null: false + t.bigint "byte_size", null: false + t.string "checksum" + t.datetime "created_at", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + end + create_table "active_storage_file_blobs", force: :cascade do |t| t.string "key" t.binary "data" @@ -23,6 +45,12 @@ t.index ["key"], name: "index_active_storage_file_blobs_on_key", unique: true end + create_table "active_storage_variant_records", force: :cascade do |t| + t.bigint "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + end + create_table "messages", force: :cascade do |t| t.string "subject" t.string "from" @@ -38,4 +66,7 @@ t.index ["body"], name: "index_messages_on_body", opclass: :gin_trgm_ops, using: :gin t.index ["list_id", "list_seq"], name: "index_messages_on_list_id_and_list_seq", unique: true end + + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" end From fe1c3aa30725de1cb143464d1391e250972f28f5 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 14 Oct 2025 23:34:30 +0900 Subject: [PATCH 09/11] Message.has_many_attached :attachments --- app/models/message.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/message.rb b/app/models/message.rb index 7480ebf..df38f2d 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -10,6 +10,8 @@ class Message < ApplicationRecord # https://blade.ruby-lang.org/ruby-talk/410000 is not. self.skip_time_zone_conversion_for_attributes = [:published_at] + has_many_attached :attachments + attr_accessor :children class << self From 41a002bf5bf04641c4870df60653e83630ee2113 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Tue, 14 Oct 2025 23:59:44 +0900 Subject: [PATCH 10/11] Use Active Storage Database storage --- config/storage.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config/storage.yml b/config/storage.yml index 4942ab6..56f94ce 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -1,10 +1,8 @@ test: - service: Disk - root: <%= Rails.root.join("tmp/storage") %> + service: Database local: - service: Disk - root: <%= Rails.root.join("storage") %> + service: Database # Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) # amazon: From 5fdd58446c4d7a7b5bcbe2364e057f52f74e6b3d Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Wed, 22 Oct 2025 00:28:34 +0900 Subject: [PATCH 11/11] Save mail attachments --- app/models/message.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/app/models/message.rb b/app/models/message.rb index df38f2d..2eb5ec9 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -67,13 +67,18 @@ def from_mail(mail, list, list_seq) end private def handle_multipart(part) - case part.content_type.downcase - when /^text\/plain/ - (self.body ||= '') << Kconv.toutf8(part.body.raw_source) - when /^text\/html;/ - (self.html_body ||= '') << Kconv.toutf8(part.body.raw_source) + if part.attachment? + file = StringIO.new(part.decoded) + attachments.attach(io: file, filename: part.filename, content_type: part.content_type) else - puts "Unknown content_type: #{part.content_type}" + case part.content_type.downcase + when /^text\/plain/ + (self.body ||= '') << Kconv.toutf8(part.body.raw_source) + when /^text\/html;/ + (self.html_body ||= '') << Kconv.toutf8(part.body.raw_source) + else + puts "Unknown content_type: #{part.content_type}" + end end end