From df6bd740e4eb9d18f96bc5f359453ef8043e11fc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Feb 2026 19:14:59 +0000 Subject: [PATCH 1/7] Initial plan From 0c58c100df72891355587e027ffcc92adbec2ebf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Feb 2026 19:20:59 +0000 Subject: [PATCH 2/7] Add request_config JSONB field and backend support for dataset HTTP headers Co-authored-by: rajkowski <10373492+rajkowski@users.noreply.github.com> --- .../DatasetDownloadRemoteFileCommand.java | 56 ++++++++++++++++--- .../domain/model/datasets/Dataset.java | 9 +++ .../datasets/DatasetRepository.java | 5 ++ ..._20260214.1000__dataset_request_config.sql | 4 ++ 4 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 src/main/resources/database/upgrade/2026/UPGRADE_20260214.1000__dataset_request_config.sql diff --git a/src/main/java/com/simisinc/platform/application/datasets/DatasetDownloadRemoteFileCommand.java b/src/main/java/com/simisinc/platform/application/datasets/DatasetDownloadRemoteFileCommand.java index c1f961c3..8ae5ac45 100644 --- a/src/main/java/com/simisinc/platform/application/datasets/DatasetDownloadRemoteFileCommand.java +++ b/src/main/java/com/simisinc/platform/application/datasets/DatasetDownloadRemoteFileCommand.java @@ -21,6 +21,8 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; @@ -48,6 +50,40 @@ public class DatasetDownloadRemoteFileCommand { private static Log LOG = LogFactory.getLog(DatasetDownloadRemoteFileCommand.class); + /** + * Parse request configuration JSON and extract headers + * + * @param requestConfig JSON string containing request configuration + * @return Map of headers, or null if no headers + */ + private static Map parseRequestHeaders(String requestConfig) { + if (StringUtils.isBlank(requestConfig)) { + return null; + } + try { + JsonNode config = JsonCommand.fromString(requestConfig); + if (config == null || !config.has("headers")) { + return null; + } + JsonNode headersNode = config.get("headers"); + if (headersNode == null || !headersNode.isObject()) { + return null; + } + Map headers = new HashMap<>(); + headersNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + String value = entry.getValue().asText(); + if (StringUtils.isNotBlank(key) && StringUtils.isNotBlank(value)) { + headers.put(key, value); + } + }); + return headers.isEmpty() ? null : headers; + } catch (Exception e) { + LOG.warn("Error parsing request config: " + e.getMessage()); + return null; + } + } + public static boolean handleRemoteFileDownload(Dataset dataset, long userId) throws DataException { if (StringUtils.isBlank(dataset.getSourceUrl())) { throw new DataException("A source url is required"); @@ -70,6 +106,8 @@ public static boolean handleRemoteFileDownload(Dataset dataset, long userId) thr // Download the file File tempFile = new File(filesystemPath); + // Parse request configuration for headers + Map requestHeaders = parseRequestHeaders(dataset.getRequestConfig()); try { if ("application/vnd.api+json".equals(fileType)) { File result = PERLSCourseListCommand.retrieveCourseListToFile(tempFile); @@ -80,13 +118,13 @@ public static boolean handleRemoteFileDownload(Dataset dataset, long userId) thr } else { // Determine if there could be multiple JSON files if (StringUtils.isNotBlank(dataset.getPagingUrlPath())) { - if (!downloadPagedFile(dataset.getSourceUrl(), dataset.getPagingUrlPath(), dataset.getRecordsPath(), + if (!downloadPagedFile(dataset.getSourceUrl(), requestHeaders, dataset.getPagingUrlPath(), dataset.getRecordsPath(), tempFile)) { throw new DataException("File with paging download error from: " + dataset.getSourceUrl()); } } else { // Download a single JSON file - if (!HttpDownloadFileCommand.execute(dataset.getSourceUrl(), tempFile)) { + if (!HttpDownloadFileCommand.execute(dataset.getSourceUrl(), requestHeaders, tempFile)) { throw new DataException("File download error from: " + dataset.getSourceUrl()); } } @@ -159,14 +197,16 @@ public static boolean handleRemoteFileDownload(Dataset dataset, long userId) thr * Downloads a series of JSON files into a single merged file * * @param url + * @param headers * @param jsonPagingPath + * @param jsonRecordsPath * @param tempFile * @return */ - public static boolean downloadPagedFile(String url, String jsonPagingPath, String jsonRecordsPath, File tempFile) { + public static boolean downloadPagedFile(String url, Map headers, String jsonPagingPath, String jsonRecordsPath, File tempFile) { // Download the first file, as a string - String content = HttpGetCommand.execute(url); + String content = HttpGetCommand.execute(url, headers); if (StringUtils.isBlank(content)) { return false; } @@ -194,7 +234,7 @@ public static boolean downloadPagedFile(String url, String jsonPagingPath, Strin } // Append any pages - appendNextUrls(jsonRecordsNode, json, jsonPagingPath, jsonRecordsPath); + appendNextUrls(jsonRecordsNode, json, headers, jsonPagingPath, jsonRecordsPath); // Write the whole JSON to a file SaveTextFileCommand.save(json.toPrettyString(), tempFile); @@ -206,7 +246,7 @@ public static boolean downloadPagedFile(String url, String jsonPagingPath, Strin } } - private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJson, String jsonPagingPath, + private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJson, Map headers, String jsonPagingPath, String jsonRecordsPath) throws IOException { if (currentJson == null) { @@ -232,7 +272,7 @@ private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJso LOG.debug("Next url: " + nextUrl); // Use the url to get the next page content - String content = HttpGetCommand.execute(nextUrl); + String content = HttpGetCommand.execute(nextUrl, headers); if (StringUtils.isBlank(content)) { throw new IOException("Content is blank"); } @@ -266,7 +306,7 @@ private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJso } // Keep going - appendNextUrls(jsonRecordsNode, nextJson, jsonPagingPath, jsonRecordsPath); + appendNextUrls(jsonRecordsNode, nextJson, headers, jsonPagingPath, jsonRecordsPath); } } diff --git a/src/main/java/com/simisinc/platform/domain/model/datasets/Dataset.java b/src/main/java/com/simisinc/platform/domain/model/datasets/Dataset.java index 35d1b241..f5b62818 100644 --- a/src/main/java/com/simisinc/platform/domain/model/datasets/Dataset.java +++ b/src/main/java/com/simisinc/platform/domain/model/datasets/Dataset.java @@ -52,6 +52,7 @@ public class Dataset extends Entity { private String fileHash = null; private String webPath = null; private Timestamp lastDownload = null; + private String requestConfig = null; // File Details private String recordsPath = null; private String pagingUrlPath = null; @@ -324,6 +325,14 @@ public void setLastDownload(Timestamp lastDownload) { this.lastDownload = lastDownload; } + public String getRequestConfig() { + return requestConfig; + } + + public void setRequestConfig(String requestConfig) { + this.requestConfig = requestConfig; + } + public String getUniqueColumnName() { return uniqueColumnName; } diff --git a/src/main/java/com/simisinc/platform/infrastructure/persistence/datasets/DatasetRepository.java b/src/main/java/com/simisinc/platform/infrastructure/persistence/datasets/DatasetRepository.java index 297351d6..3724bc9a 100644 --- a/src/main/java/com/simisinc/platform/infrastructure/persistence/datasets/DatasetRepository.java +++ b/src/main/java/com/simisinc/platform/infrastructure/persistence/datasets/DatasetRepository.java @@ -134,6 +134,8 @@ public static Dataset add(Dataset record) { .add("modified_by", record.getModifiedBy()); insertValues.add( new SqlValue("column_config", SqlValue.JSONB_TYPE, DatasetColumnJSONCommand.createColumnJSONString(record))); + insertValues.add( + new SqlValue("request_config", SqlValue.JSONB_TYPE, StringUtils.trimToNull(record.getRequestConfig()))); record.setId(DB.insertInto(TABLE_NAME, insertValues, PRIMARY_KEY)); if (record.getId() == -1) { LOG.error("An id was not set!"); @@ -162,6 +164,8 @@ public static Dataset update(Dataset record) { .add("modified_by", record.getModifiedBy()); updateValues.add( new SqlValue("column_config", SqlValue.JSONB_TYPE, DatasetColumnJSONCommand.createColumnJSONString(record))); + updateValues.add( + new SqlValue("request_config", SqlValue.JSONB_TYPE, StringUtils.trimToNull(record.getRequestConfig()))); // Where if (DB.update(TABLE_NAME, updateValues, DB.WHERE("dataset_id = ?", record.getId()))) { return record; @@ -428,6 +432,7 @@ private static Dataset buildRecord(ResultSet rs) { record.setRecordsPath(rs.getString("records_path")); record.setScheduledDate(rs.getTimestamp("scheduled_date")); record.setLastDownload(rs.getTimestamp("last_download")); + record.setRequestConfig(rs.getString("request_config")); record.setProcessStatus(DB.getInt(rs, "process_status", 0)); record.setProcessMessage(rs.getString("process_message")); record.setScheduleEnabled(rs.getBoolean("schedule_enabled")); diff --git a/src/main/resources/database/upgrade/2026/UPGRADE_20260214.1000__dataset_request_config.sql b/src/main/resources/database/upgrade/2026/UPGRADE_20260214.1000__dataset_request_config.sql new file mode 100644 index 00000000..30ffff5b --- /dev/null +++ b/src/main/resources/database/upgrade/2026/UPGRADE_20260214.1000__dataset_request_config.sql @@ -0,0 +1,4 @@ +-- Copyright 2026 Matt Rajkowski, Licensed under the Apache License, Version 2.0 + +-- Add request configuration for dataset synchronization (headers, method, OAuth) +ALTER TABLE datasets ADD COLUMN IF NOT EXISTS request_config JSONB; From bb2b01aabd7886343c0b0ab9999c7d9e45268e1e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Feb 2026 19:23:56 +0000 Subject: [PATCH 3/7] Add UI for dataset request configuration with dynamic header management Co-authored-by: rajkowski <10373492+rajkowski@users.noreply.github.com> --- .../widgets/editor/DatasetDetailsAjax.java | 9 +- .../widgets/editor/SaveDatasetAjax.java | 73 +++++++- .../widgets/editor/visual-data-editor.js | 169 ++++++++++++++++++ 3 files changed, 247 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetDetailsAjax.java b/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetDetailsAjax.java index 7bf375a5..233fa8ff 100644 --- a/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetDetailsAjax.java +++ b/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetDetailsAjax.java @@ -95,7 +95,14 @@ public WidgetContext execute(WidgetContext context) { sb.append("\"syncRecordCount\":").append(dataset.getSyncRecordCount()).append(","); sb.append("\"syncAddCount\":").append(dataset.getSyncAddCount()).append(","); sb.append("\"syncUpdateCount\":").append(dataset.getSyncUpdateCount()).append(","); - sb.append("\"syncDeleteCount\":").append(dataset.getSyncDeleteCount()); + sb.append("\"syncDeleteCount\":").append(dataset.getSyncDeleteCount()).append(","); + + // Request configuration + if (StringUtils.isNotBlank(dataset.getRequestConfig())) { + sb.append("\"requestConfig\":").append(dataset.getRequestConfig()); + } else { + sb.append("\"requestConfig\":null"); + } sb.append("}"); diff --git a/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java b/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java index f8d2be57..4fc5a42e 100644 --- a/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java +++ b/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java @@ -16,8 +16,12 @@ package com.simisinc.platform.presentation.widgets.editor; +import com.simisinc.platform.application.datasets.SaveDatasetCommand; +import com.simisinc.platform.domain.model.datasets.Dataset; +import com.simisinc.platform.infrastructure.persistence.datasets.DatasetRepository; import com.simisinc.platform.presentation.controller.WidgetContext; import com.simisinc.platform.presentation.widgets.GenericWidget; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -44,8 +48,71 @@ public WidgetContext execute(WidgetContext context) { return context; } - // Stub implementation - to be completed - context.setJson("{\"success\":true,\"message\":\"Dataset save not yet implemented\"}"); - return context; + // Get the dataset ID + long datasetId = context.getParameterAsLong("id", -1); + if (datasetId == -1) { + context.setJson("{\"success\":false,\"message\":\"Dataset id is required\"}"); + context.setSuccess(false); + return context; + } + + // Retrieve the dataset + Dataset dataset = DatasetRepository.findById(datasetId); + if (dataset == null) { + context.setJson("{\"success\":false,\"message\":\"Dataset not found\"}"); + context.setSuccess(false); + return context; + } + + // Update fields from parameters + String name = context.getParameter("name"); + if (StringUtils.isNotBlank(name)) { + dataset.setName(name); + } + + String sourceUrl = context.getParameter("sourceUrl"); + if (sourceUrl != null) { + dataset.setSourceUrl(StringUtils.trimToNull(sourceUrl)); + } + + String sourceInfo = context.getParameter("sourceInfo"); + if (sourceInfo != null) { + dataset.setSourceInfo(StringUtils.trimToNull(sourceInfo)); + } + + String requestConfig = context.getParameter("requestConfig"); + if (requestConfig != null) { + dataset.setRequestConfig(StringUtils.trimToNull(requestConfig)); + } + + String syncEnabled = context.getParameter("syncEnabled"); + if (syncEnabled != null) { + dataset.setSyncEnabled("true".equalsIgnoreCase(syncEnabled)); + } + + String scheduleEnabled = context.getParameter("scheduleEnabled"); + if (scheduleEnabled != null) { + dataset.setScheduleEnabled("true".equalsIgnoreCase(scheduleEnabled)); + } + + // Set the user who is modifying the dataset + dataset.setModifiedBy(context.getUserId()); + + // Save the dataset + try { + Dataset savedDataset = SaveDatasetCommand.saveDataset(dataset); + if (savedDataset == null) { + context.setJson("{\"success\":false,\"message\":\"Failed to save dataset\"}"); + context.setSuccess(false); + return context; + } + context.setJson("{\"success\":true,\"message\":\"Dataset saved successfully\",\"id\":" + savedDataset.getId() + "}"); + return context; + } catch (Exception e) { + LOG.error("Error saving dataset", e); + context.setJson("{\"success\":false,\"message\":\"Error: " + e.getMessage() + "\"}"); + context.setSuccess(false); + return context; + } } } diff --git a/src/main/webapp/javascript/widgets/editor/visual-data-editor.js b/src/main/webapp/javascript/widgets/editor/visual-data-editor.js index 04914c28..89eaaeee 100644 --- a/src/main/webapp/javascript/widgets/editor/visual-data-editor.js +++ b/src/main/webapp/javascript/widgets/editor/visual-data-editor.js @@ -1480,6 +1480,19 @@ class VisualDataEditor { +
+
Request Configuration
+

Configure HTTP request headers and authentication for data source access.

+ +
+ ${this.renderRequestHeaders(dataset)} +
+ + +
+
+ + +
+ `).join(''); + } + + renderEditRequestHeaders(dataset) { + let headers = {}; + + // Parse the requestConfig JSON if it exists + if (dataset.requestConfig) { + try { + const config = JSON.parse(dataset.requestConfig); + if (config && config.headers) { + headers = config.headers; + } + } catch (e) { + console.warn('Failed to parse request config:', e); + } + } + + // Render header rows + const headerEntries = Object.entries(headers); + if (headerEntries.length === 0) { + return '
No headers configured
'; + } + + return headerEntries.map(([key, value]) => ` +
+ + + +
+ `).join(''); + } + + addRequestHeader(containerId) { + const container = document.getElementById(containerId); + if (!container) return; + + // Remove empty message if present + const emptyMsg = container.querySelector('.empty-headers'); + if (emptyMsg) { + emptyMsg.remove(); + } + + // Add new header row + const headerRow = document.createElement('div'); + headerRow.className = 'header-row'; + headerRow.style.cssText = 'display: flex; gap: 10px; margin-bottom: 8px; align-items: center;'; + headerRow.innerHTML = ` + + + + `; + container.appendChild(headerRow); + + // Setup remove listener for new row + headerRow.querySelector('.remove-header-btn').addEventListener('click', () => { + headerRow.remove(); + // Show empty message if no headers left + if (container.querySelectorAll('.header-row').length === 0) { + container.innerHTML = '
No headers configured
'; + } + }); + } + + setupRemoveHeaderListeners(containerId) { + const container = document.getElementById(containerId); + if (!container) return; + + container.querySelectorAll('.remove-header-btn').forEach(btn => { + btn.addEventListener('click', () => { + btn.closest('.header-row').remove(); + // Show empty message if no headers left + if (container.querySelectorAll('.header-row').length === 0) { + container.innerHTML = '
No headers configured
'; + } + }); + }); + } + + collectRequestHeaders(containerId) { + const container = document.getElementById(containerId); + if (!container) return null; + + const headers = {}; + container.querySelectorAll('.header-row').forEach(row => { + const key = row.querySelector('.header-key').value.trim(); + const value = row.querySelector('.header-value').value.trim(); + if (key && value) { + headers[key] = value; + } + }); + + return Object.keys(headers).length > 0 ? { headers } : null; + } + renderMappingRows(dataset) { const mappings = dataset.fieldMappings || []; if (mappings.length === 0) { @@ -1777,6 +1918,18 @@ class VisualDataEditor { + +
+ +

Add HTTP headers for authentication or custom request configuration.

+
+ ${this.renderEditRequestHeaders(item)} +
+ +
+
+
+ Request Headers +

Configure HTTP request headers for authentication or custom request configuration (e.g., Authorization tokens, API keys).

+ "/> +
+ +
+ +
+ <%--
Download on a schedule? From c0bb544600cfb8c09f419f6c3418fe0d5e82c057 Mon Sep 17 00:00:00 2001 From: Matt Rajkowski Date: Sun, 15 Feb 2026 16:16:15 -0500 Subject: [PATCH 6/7] Set bean value --- .../platform/application/datasets/SaveDatasetCommand.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/simisinc/platform/application/datasets/SaveDatasetCommand.java b/src/main/java/com/simisinc/platform/application/datasets/SaveDatasetCommand.java index 8173a25e..bc0fccfb 100644 --- a/src/main/java/com/simisinc/platform/application/datasets/SaveDatasetCommand.java +++ b/src/main/java/com/simisinc/platform/application/datasets/SaveDatasetCommand.java @@ -85,6 +85,7 @@ public static Dataset saveDataset(Dataset datasetBean) throws DataException { dataset.setSourceInfo(datasetBean.getSourceInfo()); dataset.setLastDownload(datasetBean.getLastDownload()); dataset.setRecordsPath(datasetBean.getRecordsPath()); + dataset.setRequestConfig(datasetBean.getRequestConfig()); return DatasetRepository.save(dataset); } } From 12c7585cef70ba77dbee6ab4fc6fa801b060dbab Mon Sep 17 00:00:00 2001 From: Matt Rajkowski Date: Sun, 15 Feb 2026 16:45:19 -0500 Subject: [PATCH 7/7] Add request header fields --- .../admin/datasets/DatasetUploadWidget.java | 10 ++ .../widgets/editor/DatasetImportAjax.java | 77 +++++++++- .../widgets/editor/DatasetSyncAjax.java | 2 +- .../widgets/editor/SaveDatasetAjax.java | 2 +- .../WEB-INF/jsp/admin/dataset-upload-form.jsp | 136 +++++++++++++++++- .../WEB-INF/jsp/cms/visual-data-editor.jsp | 11 ++ .../widgets/editor/visual-data-editor.js | 90 +++++++++++- 7 files changed, 319 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/simisinc/platform/presentation/widgets/admin/datasets/DatasetUploadWidget.java b/src/main/java/com/simisinc/platform/presentation/widgets/admin/datasets/DatasetUploadWidget.java index ecf6b6a1..e06fe3dd 100644 --- a/src/main/java/com/simisinc/platform/presentation/widgets/admin/datasets/DatasetUploadWidget.java +++ b/src/main/java/com/simisinc/platform/presentation/widgets/admin/datasets/DatasetUploadWidget.java @@ -61,6 +61,7 @@ public WidgetContext post(WidgetContext context) { String sourceUrl = context.getParameter("sourceUrl"); String fileType = context.getParameter("fileType"); String sourceInfo = context.getParameter("sourceInfo"); + String requestConfig = context.getParameter("requestConfig"); // Populate the fields Dataset datasetBean = new Dataset(); @@ -73,6 +74,9 @@ public WidgetContext post(WidgetContext context) { if (StringUtils.isNotBlank(sourceInfo)) { datasetBean.setSourceInfo(StringUtils.trimToNull(sourceInfo)); } + if (StringUtils.isNotBlank(requestConfig)) { + datasetBean.setRequestConfig(StringUtils.trimToNull(requestConfig)); + } datasetBean.setFileType(fileType); LOG.info("fileType: " + fileType); @@ -95,6 +99,12 @@ public WidgetContext post(WidgetContext context) { } } + if (datasetBean.getId() == -1) { + context.setWarningMessage("The dataset record was not saved"); + context.setRequestObject(datasetBean); + return context; + } + LOG.info("New dataset id... " + datasetBean.getId()); context.setRedirect("/admin/dataset-preview?datasetId=" + datasetBean.getId()); return context; diff --git a/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetImportAjax.java b/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetImportAjax.java index 0d49afec..b6d0423a 100644 --- a/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetImportAjax.java +++ b/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetImportAjax.java @@ -16,8 +16,13 @@ package com.simisinc.platform.presentation.widgets.editor; +import com.simisinc.platform.application.DataException; +import com.simisinc.platform.application.datasets.DatasetDownloadRemoteFileCommand; +import com.simisinc.platform.application.datasets.DatasetUploadFileCommand; +import com.simisinc.platform.domain.model.datasets.Dataset; import com.simisinc.platform.presentation.controller.WidgetContext; import com.simisinc.platform.presentation.widgets.GenericWidget; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,7 +37,7 @@ public class DatasetImportAjax extends GenericWidget { static final long serialVersionUID = -8484048371911908899L; private static Log LOG = LogFactory.getLog(DatasetImportAjax.class); - public WidgetContext execute(WidgetContext context) { + public WidgetContext post(WidgetContext context) { LOG.debug("DatasetImportAjax..."); @@ -44,8 +49,74 @@ public WidgetContext execute(WidgetContext context) { return context; } - // Stub implementation - to be completed - context.setJson("{\"success\":true,\"message\":\"Dataset import not yet implemented\"}"); + // Check the form values + String name = context.getParameter("name"); + String sourceUrl = context.getParameter("sourceUrl"); + String fileType = context.getParameter("fileType"); + String requestConfig = context.getParameter("requestConfig"); + + if (StringUtils.isBlank(name)) { + context.setJson("{\"success\":false,\"message\":\"Dataset name is required\"}"); + context.setSuccess(false); + return context; + } + + // Populate the fields + Dataset datasetBean = new Dataset(); + datasetBean.setName(name); + datasetBean.setCreatedBy(context.getUserId()); + datasetBean.setModifiedBy(context.getUserId()); + + if (StringUtils.isNotBlank(sourceUrl)) { + // Remote URL import + datasetBean.setSourceUrl(sourceUrl.trim()); + if (StringUtils.isNotBlank(requestConfig)) { + datasetBean.setRequestConfig(StringUtils.trimToNull(requestConfig)); + } + if (StringUtils.isNotBlank(fileType)) { + datasetBean.setFileType(fileType); + } else { + // Default to JSON if not specified + datasetBean.setFileType("application/json"); + } + + // Download the remote file + try { + DatasetDownloadRemoteFileCommand.handleRemoteFileDownload(datasetBean, context.getUserId()); + } catch (DataException e) { + LOG.error("Error downloading remote file: " + e.getMessage()); + context.setJson("{\"success\":false,\"message\":\"" + escapeJson(e.getMessage()) + "\"}"); + context.setSuccess(false); + return context; + } + } else { + // File upload import + if (StringUtils.isNotBlank(fileType)) { + datasetBean.setFileType(fileType); + } + + // Check for an uploaded file and validate + if (!DatasetUploadFileCommand.handleUpload(context, datasetBean)) { + LOG.error("File upload failed"); + context.setJson("{\"success\":false,\"message\":\"File upload failed\"}"); + context.setSuccess(false); + return context; + } + } + + LOG.info("New dataset imported with id... " + datasetBean.getId()); + context.setJson("{\"success\":true,\"datasetId\":" + datasetBean.getId() + "}"); return context; } + + private String escapeJson(String text) { + if (text == null) { + return ""; + } + return text.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } } diff --git a/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetSyncAjax.java b/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetSyncAjax.java index 6550fa7e..36044148 100644 --- a/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetSyncAjax.java +++ b/src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetSyncAjax.java @@ -32,7 +32,7 @@ public class DatasetSyncAjax extends GenericWidget { static final long serialVersionUID = -8484048371911908901L; private static Log LOG = LogFactory.getLog(DatasetSyncAjax.class); - public WidgetContext execute(WidgetContext context) { + public WidgetContext post(WidgetContext context) { LOG.debug("DatasetSyncAjax..."); diff --git a/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java b/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java index 4fc5a42e..96f5f204 100644 --- a/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java +++ b/src/main/java/com/simisinc/platform/presentation/widgets/editor/SaveDatasetAjax.java @@ -36,7 +36,7 @@ public class SaveDatasetAjax extends GenericWidget { static final long serialVersionUID = -8484048371911908898L; private static Log LOG = LogFactory.getLog(SaveDatasetAjax.class); - public WidgetContext execute(WidgetContext context) { + public WidgetContext post(WidgetContext context) { LOG.debug("SaveDatasetAjax..."); diff --git a/src/main/webapp/WEB-INF/jsp/admin/dataset-upload-form.jsp b/src/main/webapp/WEB-INF/jsp/admin/dataset-upload-form.jsp index 0c7446cd..79a74e9e 100644 --- a/src/main/webapp/WEB-INF/jsp/admin/dataset-upload-form.jsp +++ b/src/main/webapp/WEB-INF/jsp/admin/dataset-upload-form.jsp @@ -15,15 +15,137 @@ --%> <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> <%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<%@ taglib prefix="web" uri="/WEB-INF/tlds/web.tld" %> + +
<%-- Required by controller --%> <%-- Form values --%> + "/> <%-- Title and Message block --%>

@@ -37,8 +159,20 @@ + + + <%----%>