From acd9a067ad712fb814f3807e16dd2253eafeb19a Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Tue, 10 Jun 2025 10:50:33 +0900 Subject: [PATCH 1/6] extract metadata from redirect headers to avoid redundant WB req --- addons/base/views.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/addons/base/views.py b/addons/base/views.py index acc7cb1f9ab..3c5592860fb 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -13,6 +13,7 @@ from flask import redirect from flask import request import furl +import json import jwe import jwt import waffle @@ -969,9 +970,17 @@ def addon_view_or_download_file(auth, path, provider, **kwargs): # TODO clean up these urls and unify what is used as a version identifier if request.method == 'HEAD': - return make_response(('', http_status.HTTP_302_FOUND, { + headers = { 'Location': file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render')) - })) + } + if hasattr(version, 'metadata') and version.metadata: + try: + # Convert metadata to JSON string and add to header + metadata_json = json.dumps(version.metadata, default=str) + headers['X-File-Metadata'] = metadata_json + except (TypeError, ValueError) as e: + logger.warning(f"Failed to serialize metadata to JSON: {e}") + return make_response(('', http_status.HTTP_302_FOUND, headers)) if action == 'download': format = extras.get('format') @@ -1088,6 +1097,7 @@ def addon_view_file(auth, node, file_node, version): 'mode': 'render', 'action': 'download', 'public_file': node.is_public, + 'version': 'Latest', }) ) From 314d5af79caef74b7545fdac063bdaf62d2f87a8 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Wed, 11 Jun 2025 09:28:12 +0900 Subject: [PATCH 2/6] add region field to NodeSettings --- .../s3/migrations/0005_nodesettings_region.py | 20 ++++++++ .../s3/migrations/0006_auto_20250610_1315.py | 46 +++++++++++++++++++ addons/s3/models.py | 5 +- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 addons/s3/migrations/0005_nodesettings_region.py create mode 100644 addons/s3/migrations/0006_auto_20250610_1315.py diff --git a/addons/s3/migrations/0005_nodesettings_region.py b/addons/s3/migrations/0005_nodesettings_region.py new file mode 100644 index 00000000000..236e0b7160a --- /dev/null +++ b/addons/s3/migrations/0005_nodesettings_region.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.28 on 2025-06-10 13:14 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('addons_s3', '0004_rename_deleted_field'), + ] + + operations = [ + migrations.AddField( + model_name='nodesettings', + name='region', + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/addons/s3/migrations/0006_auto_20250610_1315.py b/addons/s3/migrations/0006_auto_20250610_1315.py new file mode 100644 index 00000000000..e6bae0a53a7 --- /dev/null +++ b/addons/s3/migrations/0006_auto_20250610_1315.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.28 on 2025-06-10 13:15 +from __future__ import unicode_literals + +from django.db import migrations +from addons.s3.utils import get_bucket_location_or_error +from addons.s3.settings import BUCKET_LOCATIONS + + +def populate_region_field(apps, schema_editor): + NodeSettings = apps.get_model('addons_s3', 'NodeSettings') + + for node_setting in NodeSettings.objects.filter(folder_id__isnull=False): + try: + if node_setting.external_account: + bucket_location = get_bucket_location_or_error( + node_setting.external_account.oauth_key, + node_setting.external_account.oauth_secret, + node_setting.folder_id + ) + + node_setting.region = bucket_location + node_setting.save() + + except Exception as e: + print(f"Failed to get region for bucket {node_setting.folder_id}: {e}") + node_setting.region = 'us-east-1' # default region + node_setting.save() + + +def reverse_populate_region_field(apps, schema_editor): + NodeSettings = apps.get_model('addons_s3', 'NodeSettings') + NodeSettings.objects.update(region=None) + +class Migration(migrations.Migration): + + dependencies = [ + ('addons_s3', '0005_nodesettings_region'), + ] + + operations = [ + migrations.RunPython( + populate_region_field, + reverse_populate_region_field + ), + ] diff --git a/addons/s3/models.py b/addons/s3/models.py index b42a1552696..0ff299d4ea3 100644 --- a/addons/s3/models.py +++ b/addons/s3/models.py @@ -46,6 +46,7 @@ class NodeSettings(BaseOAuthNodeSettings, BaseStorageAddon): folder_name = models.TextField(blank=True, null=True) encrypt_uploads = models.BooleanField(default=ENCRYPT_UPLOADS_DEFAULT) user_settings = models.ForeignKey(UserSettings, null=True, blank=True, on_delete=models.CASCADE) + region = models.CharField(max_length=50, blank=True, null=True) @property def folder_path(self): @@ -68,6 +69,7 @@ def set_folder(self, folder_id, auth): self.external_account.oauth_secret, folder_id ) + self.region = bucket_location try: bucket_location = BUCKET_LOCATIONS[bucket_location] except KeyError: @@ -139,7 +141,8 @@ def serialize_waterbutler_settings(self): raise exceptions.AddonError('Cannot serialize settings for S3 addon') return { 'bucket': self.folder_id, - 'encrypt_uploads': self.encrypt_uploads + 'encrypt_uploads': self.encrypt_uploads, + 'region': self.region } def create_waterbutler_log(self, auth, action, metadata): From 5fcbcea963cc86a87526225e7f28b600203e50aa Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 4 Jul 2025 08:56:36 +0900 Subject: [PATCH 3/6] add s3 configure --- docker-compose.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cff0bf911f8..a0b050597a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,12 @@ services: - 9200:9200 volumes: - elasticsearch_data_vol:/usr/share/elasticsearch/data + environment: + - "ES_JAVA_OPTS=-Des.bootstrap.system_call_filter=false" + ulimits: + memlock: + soft: -1 + hard: -1 stdin_open: true # Temporary: Remove when we've upgraded to ES6 @@ -67,6 +73,13 @@ services: - 9201:9200 volumes: - elasticsearch6_data_vol:/usr/share/elasticsearch/data + environment: + - "discovery.type=single-node" + - "ES_JAVA_OPTS=-Des.bootstrap.system_call_filter=false" + ulimits: + memlock: + soft: -1 + hard: -1 stdin_open: true postgres: From 246c6e00802b09e74ed3066ed4da9c6dc8263dcf Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Fri, 4 Jul 2025 09:01:55 +0900 Subject: [PATCH 4/6] add s3 configure --- docker-compose.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a0b050597a9..cff0bf911f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,12 +58,6 @@ services: - 9200:9200 volumes: - elasticsearch_data_vol:/usr/share/elasticsearch/data - environment: - - "ES_JAVA_OPTS=-Des.bootstrap.system_call_filter=false" - ulimits: - memlock: - soft: -1 - hard: -1 stdin_open: true # Temporary: Remove when we've upgraded to ES6 @@ -73,13 +67,6 @@ services: - 9201:9200 volumes: - elasticsearch6_data_vol:/usr/share/elasticsearch/data - environment: - - "discovery.type=single-node" - - "ES_JAVA_OPTS=-Des.bootstrap.system_call_filter=false" - ulimits: - memlock: - soft: -1 - hard: -1 stdin_open: true postgres: From b4d5a9d4f834df2e73169df4e7d2317207341268 Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Mon, 13 Oct 2025 16:44:18 +0900 Subject: [PATCH 5/6] update files for flake8 --- addons/base/views.py | 2 +- addons/s3/migrations/0006_auto_20250610_1315.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/addons/base/views.py b/addons/base/views.py index 3c5592860fb..4f0f577f351 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -979,7 +979,7 @@ def addon_view_or_download_file(auth, path, provider, **kwargs): metadata_json = json.dumps(version.metadata, default=str) headers['X-File-Metadata'] = metadata_json except (TypeError, ValueError) as e: - logger.warning(f"Failed to serialize metadata to JSON: {e}") + logger.warning(f'Failed to serialize metadata to JSON: {e}') return make_response(('', http_status.HTTP_302_FOUND, headers)) if action == 'download': diff --git a/addons/s3/migrations/0006_auto_20250610_1315.py b/addons/s3/migrations/0006_auto_20250610_1315.py index e6bae0a53a7..5bf3d5ac81d 100644 --- a/addons/s3/migrations/0006_auto_20250610_1315.py +++ b/addons/s3/migrations/0006_auto_20250610_1315.py @@ -9,7 +9,7 @@ def populate_region_field(apps, schema_editor): NodeSettings = apps.get_model('addons_s3', 'NodeSettings') - + for node_setting in NodeSettings.objects.filter(folder_id__isnull=False): try: if node_setting.external_account: @@ -18,12 +18,12 @@ def populate_region_field(apps, schema_editor): node_setting.external_account.oauth_secret, node_setting.folder_id ) - + node_setting.region = bucket_location node_setting.save() - + except Exception as e: - print(f"Failed to get region for bucket {node_setting.folder_id}: {e}") + print(f'Failed to get region for bucket {node_setting.folder_id}: {e}') node_setting.region = 'us-east-1' # default region node_setting.save() From 6b9df8d36a08d041f055ee8d512bc914e29d9d2a Mon Sep 17 00:00:00 2001 From: TOMONORI ENDOU Date: Mon, 13 Oct 2025 17:05:39 +0900 Subject: [PATCH 6/6] update message --- addons/s3/migrations/0006_auto_20250610_1315.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/s3/migrations/0006_auto_20250610_1315.py b/addons/s3/migrations/0006_auto_20250610_1315.py index 5bf3d5ac81d..3cdfb7a80b8 100644 --- a/addons/s3/migrations/0006_auto_20250610_1315.py +++ b/addons/s3/migrations/0006_auto_20250610_1315.py @@ -24,7 +24,7 @@ def populate_region_field(apps, schema_editor): except Exception as e: print(f'Failed to get region for bucket {node_setting.folder_id}: {e}') - node_setting.region = 'us-east-1' # default region + node_setting.region = 'us-east-1' # aws default region node_setting.save()