Verify htapi

google/cloud/odbc/bq_driver/internal/data_translation.cc

@@ -2011,6 +2011,7 @@ StatusRecord ConvertBytesToChar(DSValue const& conn_val,
     status_record =
         StatusRecord{SQLStates::k_01004(), "String data, right truncated"};
   } else {
+    std::memset(dest, 0, dest_data.buflen);
     std::memcpy(dest, conn_val.data(), conn_val.size());
     if (dest_data.result_len) {
       *dest_data.result_len = conn_val.size();

google/cloud/odbc/bq_driver/internal/odbc_internal_commons.cc

@@ -1290,6 +1290,141 @@ odbc_internal::StatusRecordOr<std::string> GetMissingAttributesStr(
   return StatusRecord::Ok();
 }
 
+#if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW)
+std::string FormatArrowTypeToString(std::shared_ptr<arrow::Array> const& column,
+                                    int64_t row, std::string const& data) {
+  auto list_type = std::static_pointer_cast<arrow::ListType>(column->type());
+  auto type_id = list_type->value_type()->id();
+  auto list_arr = std::static_pointer_cast<arrow::ListArray>(column);
+  int64_t start = list_arr->value_offset(row);
+  int64_t length = list_arr->value_length(row);
+
+  std::string value = "[";
+
+  auto add_sep = [&value](int64_t i) {
+    if (i > 0) value += ", ";
+  };
+
+  switch (type_id) {
+    case arrow::Type::BINARY: {
+      auto bin_arr =
+          std::static_pointer_cast<arrow::BinaryArray>(list_arr->values());
+      for (int64_t i = 0; i < length; ++i) {
+        add_sep(i);
+        std::string bytes = bin_arr->GetString(start + i);
+        std::string base64 =
+            Base64Encode(reinterpret_cast<uint8_t const*>(bytes.data()),
+                         static_cast<int>(bytes.size()));
+        value += base64;
+      }
+      break;
+    }
+    case arrow::Type::STRUCT: {
+      auto struct_type =
+          std::static_pointer_cast<arrow::StructType>(list_type->value_type());
+
+      auto const& fields = struct_type->fields();
+
+      bool is_range = fields.size() == 2 && fields[0]->name() == "start" &&
+                      fields[1]->name() == "end";
+
+      auto struct_arr =
+          std::static_pointer_cast<arrow::StructArray>(list_arr->values());
+
+      for (int64_t i = 0; i < length; ++i) {
+        add_sep(i);
+
+        value += is_range ? ArrowRangeToString(struct_arr, start + i)
+                          : ArrowStructToString(struct_arr, start + i);
+      }
+      break;
+    }
+    case arrow::Type::DECIMAL128:
+    case arrow::Type::DECIMAL256: {
+      auto format_decimal_array = [&](auto const& decimal_arr) {
+        for (int64_t i = 0; i < length; ++i) {
+          add_sep(i);
+
+          auto scalar = decimal_arr->GetScalar(start + i).ValueOrDie();
+
+          value += TrimTrailingZeros(scalar->ToString());
+        }
+        return value;
+      };
+
+      if (type_id == arrow::Type::DECIMAL128) {
+        auto decimal_arr = std::static_pointer_cast<arrow::Decimal128Array>(
+            list_arr->values());
+
+        return format_decimal_array(decimal_arr);
+      } else {
+        auto decimal_arr = std::static_pointer_cast<arrow::Decimal256Array>(
+            list_arr->values());
+
+        return format_decimal_array(decimal_arr);
+      }
+      break;
+    }
+    case arrow::Type::TIMESTAMP: {
+      auto timestamp_arr =
+          std::static_pointer_cast<arrow::TimestampArray>(list_arr->values());
+      for (int64_t i = 0; i < length; ++i) {
+        add_sep(i);
+        auto scalar = timestamp_arr->GetScalar(start + i).ValueOrDie();
+        auto time_struct = ConvertStringToTimestampStruct(scalar->ToString());
+        if (!time_struct) {
+          return time_struct.GetStatusRecord().message;
+        }
+
+        if (scalar->ToString().back() != 'Z') {
+          value += FormatDatetimeToString(*time_struct);
+        } else {
+          value += FormatTimestampToString(*time_struct);
+        }
+      }
+      break;
+    }
+    case arrow::Type::DATE32: {
+      auto date_arr =
+          std::static_pointer_cast<arrow::Date32Array>(list_arr->values());
+      for (int64_t i = 0; i < length; ++i) {
+        add_sep(i);
+        auto scalar = date_arr->GetScalar(start + i).ValueOrDie();
+        auto date_struct = ConvertStringToDateStruct(scalar->ToString());
+
+        auto date_str = FormatDateToString(*date_struct);
+        value += *date_str;
+      }
+      break;
+    }
+    case arrow::Type::TIME64: {
+      auto time_arr =
+          std::static_pointer_cast<arrow::Time64Array>(list_arr->values());
+      for (int64_t i = 0; i < length; ++i) {
+        add_sep(i);
+        auto scalar = time_arr->GetScalar(start + i).ValueOrDie();
+        auto time_struct = ConvertToTimeStruct(scalar->ToString());
+        auto time_str = FormatTimetoString(time_struct);
+        value += time_str;
+      }
+      break;
+    }
+    default: {
+      value = data;
+
+      if (value.rfind("list<", 0) == 0) {
+        auto pos = value.find('[');
+        if (pos != std::string::npos) {
+          value = value.substr(pos);
+        }
+      }
+      return value;
+    }
+  }
+  return value + "]";
+}
+#endif  // (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW)
+
 odbc_internal::StatusRecord ValidateAllowedAttributes(
     ConnectionHandle* conn_handle, Section const& attributes) {
   StatusRecord status_record = StatusRecord::Ok();

google/cloud/odbc/bq_driver/internal/odbc_internal_commons.h

@@ -25,6 +25,7 @@
 #include "absl/types/variant.h"
 #if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW)
 #include <arrow/api.h>
+#include <arrow/array.h>
 #include <arrow/io/memory.h>
 #include <arrow/ipc/api.h>
 #include <arrow/ipc/reader.h>
@@ -323,6 +324,35 @@ inline void TimestampToDSValue(const SQL_TIMESTAMP_STRUCT& timestamp,
   std::memcpy(value.data(), &timestamp, sizeof(SQL_TIMESTAMP_STRUCT));
 }
 
+inline std::string TrimTrailingZeros(std::string value) {
+  auto dot_pos = value.find('.');
+
+  if (dot_pos == std::string::npos) {
+    return value;
+  }
+  while (!value.empty() && value.back() == '0') {
+    value.pop_back();
+  }
+  if (!value.empty() && value.back() == '.') {
+    value.pop_back();
+  }
+  return value;
+}
+
+inline void DatetimeToDSValue(const SQL_TIMESTAMP_STRUCT& datetime,
+                              DSValue& value) {
+  std::ostringstream ss;
+  ss << std::setfill('0') << std::setw(4) << datetime.year << "-"
+     << std::setw(2) << datetime.month << "-" << std::setw(2) << datetime.day
+     << "T" << std::setw(2) << datetime.hour << ":" << std::setw(2)
+     << datetime.minute << ":" << std::setw(2) << datetime.second;
+
+  if (datetime.fraction > 0) {
+    ss << "." << std::setw(6) << datetime.fraction;
+  }
+  StringToDSValue(ss.str(), value);
+}
+
 inline void DSValueToTimestamp(DSValue const& value,
                                SQL_TIMESTAMP_STRUCT& timestamp_struct) {
   std::memcpy(&timestamp_struct, value.data(), sizeof(SQL_TIMESTAMP_STRUCT));
@@ -435,6 +465,60 @@ inline void GetSinglePrecisionInterval(
   }
 }
 
+#if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW)
+inline std::string ArrowRangeToString(
+    std::shared_ptr<arrow::StructArray> const& struct_arr, int64_t row) {
+  auto start_scalar = struct_arr->field(0)->GetScalar(row);
+  auto end_scalar = struct_arr->field(1)->GetScalar(row);
+
+  return "[" + start_scalar.ValueOrDie()->ToString() + ", " +
+         end_scalar.ValueOrDie()->ToString() + ")";
+}
+
+inline std::string ArrowStructToString(
+    std::shared_ptr<arrow::StructArray> const& struct_arr, int64_t row) {
+  std::function<std::string(std::shared_ptr<arrow::Array> const&, int64_t)>
+      format_value;
+
+  format_value = [&format_value](std::shared_ptr<arrow::Array> const& array,
+                                 int64_t row) -> std::string {
+    if (array->IsNull(row)) {
+      return "null";
+    }
+
+    if (array->type_id() == arrow::Type::STRUCT) {
+      auto struct_arr = std::static_pointer_cast<arrow::StructArray>(array);
+
+      auto struct_type =
+          std::static_pointer_cast<arrow::StructType>(struct_arr->type());
+
+      std::string result = "{";
+
+      for (int i = 0; i < struct_type->num_fields(); ++i) {
+        if (i > 0) {
+          result += ",";
+        }
+
+        result += struct_type->field(i)->name();
+        result += ":";
+        result += format_value(struct_arr->field(i), row);
+      }
+
+      result += "}";
+      return result;
+    }
+
+    auto scalar = array->GetScalar(row);
+    return scalar.ok() ? scalar.ValueOrDie()->ToString() : "null";
+  };
+  return format_value(struct_arr, row);
+}
+
+std::string FormatArrowTypeToString(std::shared_ptr<arrow::Array> const& column,
+                                    int64_t row, std::string const& data);
+
+#endif  // (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW)
+
 inline void ArrayJsonToDSValue(std::string const& str, DSValue& value,
                                BQDataType array_type) {
   nlohmann::json json_data = nlohmann::json::parse(str);

google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc

@@ -316,6 +316,11 @@ StatusRecordOr<std::shared_ptr<arrow::Schema>> GetArrowSchema(
       case arrow::Type::DECIMAL256:
         col_schema.col_type = BQDataType::kBigNumeric;
         break;
+      case arrow::Type::INTERVAL_MONTH_DAY_NANO:
+      case arrow::Type::INTERVAL_DAY_TIME:
+      case arrow::Type::INTERVAL_MONTHS:
+        col_schema.col_type = BQDataType::kInterval;
+        break;
       case arrow::Type::LIST:
         // For other datatypes within an array, we don't have any special
         // handling. Setting 'is_mode_repeated' is enough
@@ -327,7 +332,15 @@ StatusRecordOr<std::shared_ptr<arrow::Schema>> GetArrowSchema(
         col_schema.is_mode_repeated = true;
         break;
       case arrow::Type::STRUCT: {
-        col_schema.col_type = BQDataType::kString;
+        auto struct_type =
+            std::static_pointer_cast<arrow::StructType>(field->type());
+        auto fields = struct_type->fields();
+        if (fields.size() == 2 && fields[0]->name() == "start" &&
+            fields[1]->name() == "end") {
+          col_schema.col_type = BQDataType::kRange;
+          break;
+        }
+        col_schema.col_type = BQDataType::kStruct;
         break;
       }
       default:
@@ -438,12 +451,37 @@ StatusRecord ProcessRecordBatch(
           if (bin_arr->IsNull(row)) {
             result_set.rows[row][col_i] = kNullValue;
           } else {
-            StringToDSValue(bin_arr->GetString(row),
-                            result_set.rows[row][col_i]);
+            std::string bytes = bin_arr->GetString(row);
+            std::string base64 =
+                Base64Encode(reinterpret_cast<uint8_t const*>(bytes.data()),
+                             static_cast<int>(bytes.size()));
+            StringToDSValue(base64, result_set.rows[row][col_i]);
           }
         }
         break;
       }
+      case arrow::Type::STRUCT: {
+        auto struct_arr = std::static_pointer_cast<arrow::StructArray>(column);
+        auto struct_type =
+            std::static_pointer_cast<arrow::StructType>(struct_arr->type());
+
+        auto const& fields = struct_type->fields();
+        bool is_range = fields.size() == 2 && fields[0]->name() == "start" &&
+                        fields[1]->name() == "end";
+
+        for (int64_t row = 0; row < num_rows; ++row) {
+          if (struct_arr->IsNull(row)) {
+            result_set.rows[row][col_i] = kNullValue;
+            continue;
+          }
+
+          std::string value = is_range ? ArrowRangeToString(struct_arr, row)
+                                       : ArrowStructToString(struct_arr, row);
+          StringToDSValue(value, result_set.rows[row][col_i]);
+        }
+        break;
+      }
+
       // For complex types, we fall back to the existing logic but apply it
       // column-wise. We still avoid the GetScalar() overhead where possible,
       // but use ToString() to maintain compatibility with the existing parsing
@@ -473,6 +511,9 @@ StatusRecord ProcessRecordBatch(
                   ConvertStringToTimestampStruct(data);
               if (!time_struct_status)
                 return time_struct_status.GetStatusRecord();
+              if (data.back() != 'Z') {
+                result_set.row_schema[col_i].col_type = BQDataType::kDatetime;
+              }
               TimestampToDSValue(*time_struct_status, row_val);
               break;
             }
@@ -489,16 +530,14 @@ StatusRecord ProcessRecordBatch(
               break;
             }
             case arrow::Type::LIST: {
-              if (data.rfind("list<", 0) == 0) {
-                auto pos = data.find('[');
-                if (pos != std::string::npos) data = data.substr(pos);
-              }
-              StringToDSValue(data, row_val);
+              auto value = FormatArrowTypeToString(column, row, data);
+              StringToDSValue(value, row_val);
               break;
             }
             case arrow::Type::DECIMAL128:
             case arrow::Type::DECIMAL256: {
-              NumericToDSValue(data, row_val);
+              std::string trim_data = TrimTrailingZeros(data);
+              NumericToDSValue(trim_data, row_val);
               break;
             }
             default: {