diff --git a/app/models/active_storage/file_blob.rb b/app/models/active_storage/file_blob.rb new file mode 100644 index 0000000..b5ba979 --- /dev/null +++ b/app/models/active_storage/file_blob.rb @@ -0,0 +1,2 @@ +class ActiveStorage::FileBlob < ApplicationRecord +end diff --git a/app/models/message.rb b/app/models/message.rb index bd4f2c6..2eb5ec9 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -10,48 +10,83 @@ class Message < ApplicationRecord # https://blade.ruby-lang.org/ruby-talk/410000 is not. self.skip_time_zone_conversion_for_attributes = [:published_at] + has_many_attached :attachments + attr_accessor :children class << self def from_mail(mail, list, list_seq) - body = Kconv.toutf8 mail.body.raw_source - if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316])) - body.gsub!("\u0000", '') - end - if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106]) - mail.header[:subject].value.chop! - end - if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710])) - mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value - end - subject = mail.subject - subject = Kconv.toutf8 subject if subject - from = Kconv.toutf8 mail.from_address&.raw - if !from && (list.name == 'ruby-core') && (list_seq == 161) - from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R - end + new.from_mail(mail, list, list_seq) + end + end - message_id = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace + def from_mail(mail, list, list_seq) + self.list_id, self.list_seq, self.published_at = list.id, list_seq, mail.date - # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value - parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value) - parent_message_id = Message.where(list_id: list.id, message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header - if !parent_message_id && (String === mail.references) - parent_message_id = Message.where(list_id: list.id, message_id_header: mail.references).pick(:id) - end - if !parent_message_id && (Array === mail.references) - mail.references.compact.each do |ref| - break if (parent_message_id = Message.where(list_id: list.id, message_id_header: ref).pick(:id)) - end + if mail.multipart? + mail.parts.each do |p| + handle_multipart p end + else + self.body = Kconv.toutf8 mail.body.raw_source + end + + if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316])) + self.body.gsub!("\u0000", '') + end + + if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106]) + mail.header[:subject].value.chop! + end + if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710])) + mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value + end + self.subject = mail.subject + self.subject = Kconv.toutf8 subject if self.subject - new list_id: list.id, list_seq: list_seq, body: body, subject: subject, from: from, published_at: mail.date, message_id_header: message_id, parent_id: parent_message_id + self.from = Kconv.toutf8 mail.from_address&.raw + if !self.from && (list.name == 'ruby-core') && (list_seq == 161) + self.from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R end - private def extract_message_id_from_in_reply_to(header) - header && header.strip.scan(/<([^>]+)>/).flatten.first + self.message_id_header = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace + + # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value + parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value) + self.parent_id = Message.where(list_id: list.id, message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header + if !self.parent_id && (String === mail.references) + self.parent_id = Message.where(list_id: list.id, message_id_header: mail.references).pick(:id) + end + if !self.parent_id && (Array === mail.references) + mail.references.compact.each do |ref| + break if (self.parent_id = Message.where(list_id: list.id, message_id_header: ref).pick(:id)) + end end + self + end + + private def handle_multipart(part) + if part.attachment? + file = StringIO.new(part.decoded) + attachments.attach(io: file, filename: part.filename, content_type: part.content_type) + else + case part.content_type.downcase + when /^text\/plain/ + (self.body ||= '') << Kconv.toutf8(part.body.raw_source) + when /^text\/html;/ + (self.html_body ||= '') << Kconv.toutf8(part.body.raw_source) + else + puts "Unknown content_type: #{part.content_type}" + end + end + end + + private def extract_message_id_from_in_reply_to(header) + header && header.strip.scan(/<([^>]+)>/).flatten.first + end + + class << self def from_s3(list_name, list_seq, s3_client = Aws::S3::Client.new(region: BLADE_BUCKET_REGION)) obj = s3_client.get_object(bucket: BLADE_BUCKET_NAME, key: "#{list_name}/#{list_seq}") diff --git a/config/storage.yml b/config/storage.yml index 4942ab6..56f94ce 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -1,10 +1,8 @@ test: - service: Disk - root: <%= Rails.root.join("tmp/storage") %> + service: Database local: - service: Disk - root: <%= Rails.root.join("storage") %> + service: Database # Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) # amazon: diff --git a/db/migrate/20251012141446_add_html_body_to_messages.rb b/db/migrate/20251012141446_add_html_body_to_messages.rb new file mode 100644 index 0000000..2c10b6e --- /dev/null +++ b/db/migrate/20251012141446_add_html_body_to_messages.rb @@ -0,0 +1,5 @@ +class AddHtmlBodyToMessages < ActiveRecord::Migration[8.0] + def change + add_column :messages, :html_body, :text + end +end diff --git a/db/migrate/20251014140646_create_active_storage_file_blobs.rb b/db/migrate/20251014140646_create_active_storage_file_blobs.rb new file mode 100644 index 0000000..0b9bfbc --- /dev/null +++ b/db/migrate/20251014140646_create_active_storage_file_blobs.rb @@ -0,0 +1,11 @@ +class CreateActiveStorageFileBlobs < ActiveRecord::Migration[8.0] + def change + create_table :active_storage_file_blobs do |t| + t.string :key + t.binary :data + + t.timestamps + end + add_index :active_storage_file_blobs, :key, unique: true + end +end diff --git a/db/migrate/20251014141822_create_active_storage_tables.active_storage.rb b/db/migrate/20251014141822_create_active_storage_tables.active_storage.rb new file mode 100644 index 0000000..6bd8bd0 --- /dev/null +++ b/db/migrate/20251014141822_create_active_storage_tables.active_storage.rb @@ -0,0 +1,57 @@ +# This migration comes from active_storage (originally 20170806125915) +class CreateActiveStorageTables < ActiveRecord::Migration[7.0] + def change + # Use Active Record's configured type for primary and foreign keys + primary_key_type, foreign_key_type = primary_and_foreign_key_types + + create_table :active_storage_blobs, id: primary_key_type do |t| + t.string :key, null: false + t.string :filename, null: false + t.string :content_type + t.text :metadata + t.string :service_name, null: false + t.bigint :byte_size, null: false + t.string :checksum + + if connection.supports_datetime_with_precision? + t.datetime :created_at, precision: 6, null: false + else + t.datetime :created_at, null: false + end + + t.index [ :key ], unique: true + end + + create_table :active_storage_attachments, id: primary_key_type do |t| + t.string :name, null: false + t.references :record, null: false, polymorphic: true, index: false, type: foreign_key_type + t.references :blob, null: false, type: foreign_key_type + + if connection.supports_datetime_with_precision? + t.datetime :created_at, precision: 6, null: false + else + t.datetime :created_at, null: false + end + + t.index [ :record_type, :record_id, :name, :blob_id ], name: :index_active_storage_attachments_uniqueness, unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + + create_table :active_storage_variant_records, id: primary_key_type do |t| + t.belongs_to :blob, null: false, index: false, type: foreign_key_type + t.string :variation_digest, null: false + + t.index [ :blob_id, :variation_digest ], name: :index_active_storage_variant_records_uniqueness, unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + end + + private + def primary_and_foreign_key_types + config = Rails.configuration.generators + setting = config.options[config.orm][:primary_key_type] + primary_key_type = setting || :primary_key + foreign_key_type = setting || :bigint + [ primary_key_type, foreign_key_type ] + end +end diff --git a/db/schema.rb b/db/schema.rb index 9b743e7..43477c3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -15,6 +15,42 @@ enable_extension "pg_catalog.plpgsql" enable_extension "pg_trgm" + create_table "active_storage_attachments", force: :cascade do |t| + t.string "name", null: false + t.string "record_type", null: false + t.bigint "record_id", null: false + t.bigint "blob_id", null: false + t.datetime "created_at", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + end + + create_table "active_storage_blobs", force: :cascade do |t| + t.string "key", null: false + t.string "filename", null: false + t.string "content_type" + t.text "metadata" + t.string "service_name", null: false + t.bigint "byte_size", null: false + t.string "checksum" + t.datetime "created_at", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + end + + create_table "active_storage_file_blobs", force: :cascade do |t| + t.string "key" + t.binary "data" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["key"], name: "index_active_storage_file_blobs_on_key", unique: true + end + + create_table "active_storage_variant_records", force: :cascade do |t| + t.bigint "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + end + create_table "messages", force: :cascade do |t| t.string "subject" t.string "from" @@ -26,7 +62,11 @@ t.timestamptz "published_at" t.string "message_id_header" t.integer "parent_id" + t.text "html_body" t.index ["body"], name: "index_messages_on_body", opclass: :gin_trgm_ops, using: :gin t.index ["list_id", "list_seq"], name: "index_messages_on_list_id_and_list_seq", unique: true end + + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" end diff --git a/lib/active_storage/service/database_service.rb b/lib/active_storage/service/database_service.rb new file mode 100644 index 0000000..6dfd866 --- /dev/null +++ b/lib/active_storage/service/database_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'active_storage/service' + +module ActiveStorage + class Service::DatabaseService < Service + def upload(key, io, checksum: nil, **) + instrument :upload, key: key, checksum: checksum do + ActiveStorage::FileBlob.find_or_initialize_by(key: key) do + it.data = io.read + it.save! + end + end + end + + def download(key) + instrument :download, key: key do + ActiveStorage::FileBlob.where(key: key).pick(:data) + end + end + + def delete(key) + instrument :delete, key: key do + ActiveStorage::FileBlob.find_by(key: key)&.destroy + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + payload[:exist] = ActiveStorage::FileBlob.exists?(key: key) + end + end + + private + + def service_name + 'Database' + end + end +end diff --git a/test/models/active_storage_file_blob_test.rb b/test/models/active_storage_file_blob_test.rb new file mode 100644 index 0000000..038dc32 --- /dev/null +++ b/test/models/active_storage_file_blob_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class ActiveStorageFileBlobTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end