Add request header configuration to admin dataset source interface

src/main/java/com/simisinc/platform/application/datasets/DatasetDownloadRemoteFileCommand.java

@@ -21,6 +21,8 @@
 import java.sql.Timestamp;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
@@ -48,6 +50,40 @@ public class DatasetDownloadRemoteFileCommand {
 
   private static Log LOG = LogFactory.getLog(DatasetDownloadRemoteFileCommand.class);
 
+  /**
+   * Parse request configuration JSON and extract headers
+   * 
+   * @param requestConfig JSON string containing request configuration
+   * @return Map of headers, or null if no headers
+   */
+  private static Map<String, String> parseRequestHeaders(String requestConfig) {
+    if (StringUtils.isBlank(requestConfig)) {
+      return null;
+    }
+    try {
+      JsonNode config = JsonCommand.fromString(requestConfig);
+      if (config == null || !config.has("headers")) {
+        return null;
+      }
+      JsonNode headersNode = config.get("headers");
+      if (headersNode == null || !headersNode.isObject()) {
+        return null;
+      }
+      Map<String, String> headers = new HashMap<>();
+      headersNode.fields().forEachRemaining(entry -> {
+        String key = entry.getKey();
+        String value = entry.getValue().asText();
+        if (StringUtils.isNotBlank(key) && StringUtils.isNotBlank(value)) {
+          headers.put(key, value);
+        }
+      });
+      return headers.isEmpty() ? null : headers;
+    } catch (Exception e) {
+      LOG.warn("Error parsing request config: " + e.getMessage());
+      return null;
+    }
+  }
+
   public static boolean handleRemoteFileDownload(Dataset dataset, long userId) throws DataException {
     if (StringUtils.isBlank(dataset.getSourceUrl())) {
       throw new DataException("A source url is required");
@@ -70,6 +106,8 @@ public static boolean handleRemoteFileDownload(Dataset dataset, long userId) thr
 
     // Download the file
     File tempFile = new File(filesystemPath);
+    // Parse request configuration for headers
+    Map<String, String> requestHeaders = parseRequestHeaders(dataset.getRequestConfig());
     try {
       if ("application/vnd.api+json".equals(fileType)) {
         File result = PERLSCourseListCommand.retrieveCourseListToFile(tempFile);
@@ -80,13 +118,13 @@ public static boolean handleRemoteFileDownload(Dataset dataset, long userId) thr
       } else {
         // Determine if there could be multiple JSON files
         if (StringUtils.isNotBlank(dataset.getPagingUrlPath())) {
-          if (!downloadPagedFile(dataset.getSourceUrl(), dataset.getPagingUrlPath(), dataset.getRecordsPath(),
+          if (!downloadPagedFile(dataset.getSourceUrl(), requestHeaders, dataset.getPagingUrlPath(), dataset.getRecordsPath(),
               tempFile)) {
             throw new DataException("File with paging download error from: " + dataset.getSourceUrl());
           }
         } else {
           // Download a single JSON file
-          if (!HttpDownloadFileCommand.execute(dataset.getSourceUrl(), tempFile)) {
+          if (!HttpDownloadFileCommand.execute(dataset.getSourceUrl(), requestHeaders, tempFile)) {
             throw new DataException("File download error from: " + dataset.getSourceUrl());
           }
         }
@@ -159,14 +197,16 @@ public static boolean handleRemoteFileDownload(Dataset dataset, long userId) thr
    * Downloads a series of JSON files into a single merged file
    * 
    * @param url
+   * @param headers
    * @param jsonPagingPath
+   * @param jsonRecordsPath
    * @param tempFile
    * @return
    */
-  public static boolean downloadPagedFile(String url, String jsonPagingPath, String jsonRecordsPath, File tempFile) {
+  public static boolean downloadPagedFile(String url, Map<String, String> headers, String jsonPagingPath, String jsonRecordsPath, File tempFile) {
 
     // Download the first file, as a string
-    String content = HttpGetCommand.execute(url);
+    String content = HttpGetCommand.execute(url, headers);
     if (StringUtils.isBlank(content)) {
       return false;
     }
@@ -194,7 +234,7 @@ public static boolean downloadPagedFile(String url, String jsonPagingPath, Strin
       }
 
       // Append any pages
-      appendNextUrls(jsonRecordsNode, json, jsonPagingPath, jsonRecordsPath);
+      appendNextUrls(jsonRecordsNode, json, headers, jsonPagingPath, jsonRecordsPath);
 
       // Write the whole JSON to a file
       SaveTextFileCommand.save(json.toPrettyString(), tempFile);
@@ -206,7 +246,7 @@ public static boolean downloadPagedFile(String url, String jsonPagingPath, Strin
     }
   }
 
-  private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJson, String jsonPagingPath,
+  private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJson, Map<String, String> headers, String jsonPagingPath,
       String jsonRecordsPath) throws IOException {
 
     if (currentJson == null) {
@@ -232,7 +272,7 @@ private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJso
     LOG.debug("Next url: " + nextUrl);
 
     // Use the url to get the next page content
-    String content = HttpGetCommand.execute(nextUrl);
+    String content = HttpGetCommand.execute(nextUrl, headers);
     if (StringUtils.isBlank(content)) {
       throw new IOException("Content is blank");
     }
@@ -266,7 +306,7 @@ private static void appendNextUrls(JsonNode jsonRecordsNode, JsonNode currentJso
     }
 
     // Keep going
-    appendNextUrls(jsonRecordsNode, nextJson, jsonPagingPath, jsonRecordsPath);
+    appendNextUrls(jsonRecordsNode, nextJson, headers, jsonPagingPath, jsonRecordsPath);
   }
 
 }

src/main/java/com/simisinc/platform/application/datasets/SaveDatasetCommand.java

@@ -85,6 +85,7 @@ public static Dataset saveDataset(Dataset datasetBean) throws DataException {
     dataset.setSourceInfo(datasetBean.getSourceInfo());
     dataset.setLastDownload(datasetBean.getLastDownload());
     dataset.setRecordsPath(datasetBean.getRecordsPath());
+    dataset.setRequestConfig(datasetBean.getRequestConfig());
     return DatasetRepository.save(dataset);
   }
 }

src/main/java/com/simisinc/platform/domain/model/datasets/Dataset.java

@@ -52,6 +52,7 @@ public class Dataset extends Entity {
   private String fileHash = null;
   private String webPath = null;
   private Timestamp lastDownload = null;
+  private String requestConfig = null;
   // File Details
   private String recordsPath = null;
   private String pagingUrlPath = null;
@@ -324,6 +325,14 @@ public void setLastDownload(Timestamp lastDownload) {
     this.lastDownload = lastDownload;
   }
 
+  public String getRequestConfig() {
+    return requestConfig;
+  }
+
+  public void setRequestConfig(String requestConfig) {
+    this.requestConfig = requestConfig;
+  }
+
   public String getUniqueColumnName() {
     return uniqueColumnName;
   }

src/main/java/com/simisinc/platform/infrastructure/persistence/datasets/DatasetRepository.java

@@ -134,6 +134,8 @@ public static Dataset add(Dataset record) {
         .add("modified_by", record.getModifiedBy());
     insertValues.add(
         new SqlValue("column_config", SqlValue.JSONB_TYPE, DatasetColumnJSONCommand.createColumnJSONString(record)));
+    insertValues.add(
+        new SqlValue("request_config", SqlValue.JSONB_TYPE, StringUtils.trimToNull(record.getRequestConfig())));
     record.setId(DB.insertInto(TABLE_NAME, insertValues, PRIMARY_KEY));
     if (record.getId() == -1) {
       LOG.error("An id was not set!");
@@ -162,6 +164,8 @@ public static Dataset update(Dataset record) {
         .add("modified_by", record.getModifiedBy());
     updateValues.add(
         new SqlValue("column_config", SqlValue.JSONB_TYPE, DatasetColumnJSONCommand.createColumnJSONString(record)));
+    updateValues.add(
+        new SqlValue("request_config", SqlValue.JSONB_TYPE, StringUtils.trimToNull(record.getRequestConfig())));
     // Where
     if (DB.update(TABLE_NAME, updateValues, DB.WHERE("dataset_id = ?", record.getId()))) {
       return record;
@@ -428,6 +432,7 @@ private static Dataset buildRecord(ResultSet rs) {
       record.setRecordsPath(rs.getString("records_path"));
       record.setScheduledDate(rs.getTimestamp("scheduled_date"));
       record.setLastDownload(rs.getTimestamp("last_download"));
+      record.setRequestConfig(rs.getString("request_config"));
       record.setProcessStatus(DB.getInt(rs, "process_status", 0));
       record.setProcessMessage(rs.getString("process_message"));
       record.setScheduleEnabled(rs.getBoolean("schedule_enabled"));

src/main/java/com/simisinc/platform/presentation/widgets/admin/datasets/DatasetSourceWidget.java

@@ -107,6 +107,7 @@ private WidgetContext updateSettings(WidgetContext context, Dataset dataset) {
     // Check form values
     dataset.setSourceUrl(context.getParameter("sourceUrl"));
     dataset.setFileType(context.getParameter("fileType"));
+    dataset.setRequestConfig(context.getParameter("requestConfig"));
 
     // Verify there is a data source url
     if (StringUtils.isBlank(dataset.getSourceUrl()) || !UrlCommand.isUrlValid(dataset.getSourceUrl())) {
@@ -141,6 +142,7 @@ private WidgetContext downloadRemoteFileAction(WidgetContext context, Dataset da
     // Check form values
     dataset.setSourceUrl(context.getParameter("sourceUrl"));
     dataset.setFileType(context.getParameter("fileType"));
+    dataset.setRequestConfig(context.getParameter("requestConfig"));
 
     // Verify there is a data source url
     if (StringUtils.isBlank(dataset.getSourceUrl()) || !UrlCommand.isUrlValid(dataset.getSourceUrl())) {

src/main/java/com/simisinc/platform/presentation/widgets/admin/datasets/DatasetUploadWidget.java

@@ -61,6 +61,7 @@ public WidgetContext post(WidgetContext context) {
     String sourceUrl = context.getParameter("sourceUrl");
     String fileType = context.getParameter("fileType");
     String sourceInfo = context.getParameter("sourceInfo");
+    String requestConfig = context.getParameter("requestConfig");
 
     // Populate the fields
     Dataset datasetBean = new Dataset();
@@ -73,6 +74,9 @@ public WidgetContext post(WidgetContext context) {
     if (StringUtils.isNotBlank(sourceInfo)) {
       datasetBean.setSourceInfo(StringUtils.trimToNull(sourceInfo));
     }
+    if (StringUtils.isNotBlank(requestConfig)) {
+      datasetBean.setRequestConfig(StringUtils.trimToNull(requestConfig));
+    }
     datasetBean.setFileType(fileType);
     LOG.info("fileType: " + fileType);
 
@@ -95,6 +99,12 @@ public WidgetContext post(WidgetContext context) {
       }
     }
 
+    if (datasetBean.getId() == -1) {
+      context.setWarningMessage("The dataset record was not saved");
+      context.setRequestObject(datasetBean);
+      return context;
+    }
+
     LOG.info("New dataset id... " + datasetBean.getId());
     context.setRedirect("/admin/dataset-preview?datasetId=" + datasetBean.getId());
     return context;

src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetDetailsAjax.java

@@ -95,7 +95,14 @@ public WidgetContext execute(WidgetContext context) {
     sb.append("\"syncRecordCount\":").append(dataset.getSyncRecordCount()).append(",");
     sb.append("\"syncAddCount\":").append(dataset.getSyncAddCount()).append(",");
     sb.append("\"syncUpdateCount\":").append(dataset.getSyncUpdateCount()).append(",");
-    sb.append("\"syncDeleteCount\":").append(dataset.getSyncDeleteCount());
+    sb.append("\"syncDeleteCount\":").append(dataset.getSyncDeleteCount()).append(",");
+
+    // Request configuration
+    if (StringUtils.isNotBlank(dataset.getRequestConfig())) {
+      sb.append("\"requestConfig\":").append(dataset.getRequestConfig());
+    } else {
+      sb.append("\"requestConfig\":null");
+    }
 
     sb.append("}");
 

src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetImportAjax.java

@@ -16,8 +16,13 @@
 
 package com.simisinc.platform.presentation.widgets.editor;
 
+import com.simisinc.platform.application.DataException;
+import com.simisinc.platform.application.datasets.DatasetDownloadRemoteFileCommand;
+import com.simisinc.platform.application.datasets.DatasetUploadFileCommand;
+import com.simisinc.platform.domain.model.datasets.Dataset;
 import com.simisinc.platform.presentation.controller.WidgetContext;
 import com.simisinc.platform.presentation.widgets.GenericWidget;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -32,7 +37,7 @@ public class DatasetImportAjax extends GenericWidget {
   static final long serialVersionUID = -8484048371911908899L;
   private static Log LOG = LogFactory.getLog(DatasetImportAjax.class);
 
-  public WidgetContext execute(WidgetContext context) {
+  public WidgetContext post(WidgetContext context) {
 
     LOG.debug("DatasetImportAjax...");
 
@@ -44,8 +49,74 @@ public WidgetContext execute(WidgetContext context) {
       return context;
     }
 
-    // Stub implementation - to be completed
-    context.setJson("{\"success\":true,\"message\":\"Dataset import not yet implemented\"}");
+    // Check the form values
+    String name = context.getParameter("name");
+    String sourceUrl = context.getParameter("sourceUrl");
+    String fileType = context.getParameter("fileType");
+    String requestConfig = context.getParameter("requestConfig");
+
+    if (StringUtils.isBlank(name)) {
+      context.setJson("{\"success\":false,\"message\":\"Dataset name is required\"}");
+      context.setSuccess(false);
+      return context;
+    }
+
+    // Populate the fields
+    Dataset datasetBean = new Dataset();
+    datasetBean.setName(name);
+    datasetBean.setCreatedBy(context.getUserId());
+    datasetBean.setModifiedBy(context.getUserId());
+
+    if (StringUtils.isNotBlank(sourceUrl)) {
+      // Remote URL import
+      datasetBean.setSourceUrl(sourceUrl.trim());
+      if (StringUtils.isNotBlank(requestConfig)) {
+        datasetBean.setRequestConfig(StringUtils.trimToNull(requestConfig));
+      }
+      if (StringUtils.isNotBlank(fileType)) {
+        datasetBean.setFileType(fileType);
+      } else {
+        // Default to JSON if not specified
+        datasetBean.setFileType("application/json");
+      }
+
+      // Download the remote file
+      try {
+        DatasetDownloadRemoteFileCommand.handleRemoteFileDownload(datasetBean, context.getUserId());
+      } catch (DataException e) {
+        LOG.error("Error downloading remote file: " + e.getMessage());
+        context.setJson("{\"success\":false,\"message\":\"" + escapeJson(e.getMessage()) + "\"}");
+        context.setSuccess(false);
+        return context;
+      }
+    } else {
+      // File upload import
+      if (StringUtils.isNotBlank(fileType)) {
+        datasetBean.setFileType(fileType);
+      }
+
+      // Check for an uploaded file and validate
+      if (!DatasetUploadFileCommand.handleUpload(context, datasetBean)) {
+        LOG.error("File upload failed");
+        context.setJson("{\"success\":false,\"message\":\"File upload failed\"}");
+        context.setSuccess(false);
+        return context;
+      }
+    }
+
+    LOG.info("New dataset imported with id... " + datasetBean.getId());
+    context.setJson("{\"success\":true,\"datasetId\":" + datasetBean.getId() + "}");
     return context;
   }
+
+  private String escapeJson(String text) {
+    if (text == null) {
+      return "";
+    }
+    return text.replace("\\", "\\\\")
+        .replace("\"", "\\\"")
+        .replace("\n", "\\n")
+        .replace("\r", "\\r")
+        .replace("\t", "\\t");
+  }
 }

src/main/java/com/simisinc/platform/presentation/widgets/editor/DatasetSyncAjax.java

@@ -32,7 +32,7 @@ public class DatasetSyncAjax extends GenericWidget {
   static final long serialVersionUID = -8484048371911908901L;
   private static Log LOG = LogFactory.getLog(DatasetSyncAjax.class);
 
-  public WidgetContext execute(WidgetContext context) {
+  public WidgetContext post(WidgetContext context) {
 
     LOG.debug("DatasetSyncAjax...");