SK-2522: Fix Python SDK v2 issues reported in Bug Bash

skyflow/utils/_skyflow_messages.py

@@ -42,12 +42,13 @@ class Error(Enum):
         EMPTY_CREDENTIAL_FILE_PATH_IN_CONFIG = f"{error_prefix} Initialization failed. Invalid credentials for {{}} with id {{}}. Specify a valid file path."
         EMPTY_CREDENTIAL_FILE_PATH = f"{error_prefix} Initialization failed. Invalid credentials. Specify a valid file path."
         INVALID_CREDENTIAL_FILE_PATH_IN_CONFIG = f"{error_prefix} Initialization failed. Invalid credentials for {{}} with id {{}}. Expected file path to be a string."
-        INVALID_CREDENTIAL_FILE_PATH = f"{error_prefix} Initialization failed. Invalid credentials. Expected file path to be a string."
+        INVALID_CREDENTIAL_FILE_PATH = f"{error_prefix} Initialization failed. Invalid credentials. Expected file path to be a valid file path."
         EMPTY_CREDENTIALS_TOKEN_IN_CONFIG = f"{error_prefix} Initialization failed. Invalid token for {{}} with id {{}}.Specify a valid credentials token."
         EMPTY_CREDENTIALS_TOKEN = f"{error_prefix} Initialization failed. Invalid token.Specify a valid credentials token."
         INVALID_CREDENTIALS_TOKEN_IN_CONFIG = f"{error_prefix} Initialization failed. Invalid credentials token for {{}} with id {{}}. Expected token to be a string."
         INVALID_CREDENTIALS_TOKEN = f"{error_prefix} Initialization failed. Invalid credentials token. Expected token to be a string."
-        EXPIRED_TOKEN = f"${error_prefix} Initialization failed. Given token is expired. Specify a valid credentials token."
+        EXPIRED_BEARER_TOKEN = f"{error_prefix} Initialization failed. Bearer token is invalid or expired."
+        EXPIRED_TOKEN = f"{error_prefix} Initialization failed. Given token is expired. Specify a valid credentials token."
         EMPTY_API_KEY_IN_CONFIG = f"{error_prefix} Initialization failed. Invalid api key for {{}} with id {{}}.Specify a valid api key."
         EMPTY_API_KEY= f"{error_prefix} Initialization failed. Invalid api key.Specify a valid api key."
         INVALID_API_KEY_IN_CONFIG = f"{error_prefix} Initialization failed. Invalid api key for {{}} with id {{}}. Expected api key to be a string."
@@ -118,10 +119,11 @@ class Error(Enum):
 
         INVALID_IDS_TYPE = f"{error_prefix} Validation error. 'ids' has a value of type {{}}. Specify 'ids' as list."
         INVALID_REDACTION_TYPE = f"{error_prefix} Validation error. 'redaction' has a value of type {{}}. Specify 'redaction' as type Skyflow.RedactionType."
-        INVALID_COLUMN_NAME = f"{error_prefix} Validation error. 'column' has a value of type {{}}. Specify 'column' as a string."
-        INVALID_COLUMN_VALUE = f"{error_prefix} Validation error. columnValues key has a value of type {{}}. Specify columnValues key as list."
+        INVALID_COLUMN_NAME = f"{error_prefix} Validation error. column_name has a value of type {{}}. Specify 'column' as a string."
+        INVALID_COLUMN_VALUE = f"{error_prefix} Validation error. column_values key has a value of type {{}}. Specify column_values key as list."
+        INVALID_COLUMN_VALUES = f"{error_prefix} Validation error. column_values key is an empty list. Specify at least one column value when column_name is passed." 
         INVALID_FIELDS_VALUE = f"{error_prefix} Validation error. fields key has a value of type{{}}. Specify fields key as list."
-        BOTH_OFFSET_AND_LIMIT_SPECIFIED = f"${error_prefix} Validation error. Both offset and limit cannot be present at the same time"
+        BOTH_OFFSET_AND_LIMIT_SPECIFIED = f"{error_prefix} Validation error. Both offset and limit cannot be present at the same time"
         INVALID_OFF_SET_VALUE = f"{error_prefix} Validation error. offset key has a value of type {{}}. Specify offset key as integer."
         INVALID_LIMIT_VALUE = f"{error_prefix} Validation error. limit key has a value of type {{}}. Specify limit key as integer."
         INVALID_DOWNLOAD_URL_VALUE = f"{error_prefix} Validation error. download_url key has a value of type {{}}. Specify download_url key as boolean."
@@ -366,7 +368,7 @@ class ErrorLogs(Enum):
         SKYFLOW_ID_IS_REQUIRED = f"{ERROR}: [{error_prefix}] Invalid {{}} request. Skyflow Id is required."
         EMPTY_SKYFLOW_ID = f"{ERROR}: [{error_prefix}] Invalid {{}} request. Skyflow Id can not be empty."
 
-        COLUMN_VALUES_IS_REQUIRED_TOKENIZE = f"{ERROR}: [{error_prefix}] Invalid {{}} request. ColumnValues are required."
+        COLUMN_VALUES_IS_REQUIRED_TOKENIZE = f"{ERROR}: [{error_prefix}] Invalid {{}} request. column_values are required."
         EMPTY_COLUMN_GROUP_IN_COLUMN_VALUES = f"{ERROR}: [{error_prefix}] Invalid {{}} request. Column group can not be null or empty in column values at index %s2."
 
         EMPTY_QUERY= f"{ERROR}: [{error_prefix}] Invalid {{}} request. Query can not be empty."

skyflow/utils/_utils.py

@@ -106,27 +106,41 @@ def convert_detected_entity_to_entity_info(detected_entity):
 def construct_invoke_connection_request(request, connection_url, logger) -> PreparedRequest:
     url = parse_path_params(connection_url.rstrip('/'), request.path_params)
 
-    try:
-        if isinstance(request.headers, dict):
-            header = to_lowercase_keys(json.loads(
-                json.dumps(request.headers)))
-        else:
-            raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code)
-    except Exception:
-        raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code)
+    header = None
+    content_type = None
 
-    if not HttpHeader.CONTENT_TYPE.lower() in header:
-        header[HttpHeader.CONTENT_TYPE_LOWERCASE] = ContentType.JSON.value
+    if request.headers is not None:
+        try:
+            if isinstance(request.headers, dict):
+                header = to_lowercase_keys(json.loads(
+                    json.dumps(request.headers)))
+
+                content_type = header.get(HttpHeader.CONTENT_TYPE_LOWERCASE)
+            else:
+                raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code)
+        except SkyflowError:
+            raise
+        except Exception:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code)
 
-    try:
-        if isinstance(request.body, dict):
-            json_data, files = get_data_from_content_type(
-                request.body, header[HttpHeader.CONTENT_TYPE_LOWERCASE]
-            )
-        else:
+    json_data = None
+    files = {}
+
+    if request.body is not None:
+        try:
+            if isinstance(request.body, dict):
+                json_data, files = get_data_from_content_type(
+                    request.body, content_type
+                )
+            else:
+                raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_BODY.value, invalid_input_error_code)
+        except SkyflowError:
+            raise
+        except Exception as e:
             raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_BODY.value, invalid_input_error_code)
-    except Exception as e:
-        raise SkyflowError( SkyflowMessages.Error.INVALID_REQUEST_BODY.value, invalid_input_error_code)
+
+    if files and header and content_type == ContentType.FORMDATA.value:
+        header.pop(HttpHeader.CONTENT_TYPE_LOWERCASE, None)
 
     validate_invoke_connection_params(logger, request.query_params, request.path_params)
 
@@ -176,16 +190,54 @@ def render_key(parents):
 def get_data_from_content_type(data, content_type):
     converted_data = data
     files = {}
+
     if content_type == ContentType.URLENCODED.value:
         converted_data = http_build_query(data)
     elif content_type == ContentType.FORMDATA.value:
-        converted_data = r_urlencode(list(), dict(), data)
-        files = {(None, None)}
+        print("Hello")
+        converted_data = None
+        files = {}
+        for key, value in data.items():
+            files[key] = (None, str(value))
     elif content_type == ContentType.JSON.value:
         converted_data = json.dumps(data)
+    elif content_type == ContentType.XML.value or content_type == 'application/xml' or content_type == 'text/xml':
+        if isinstance(data, dict):
+            converted_data = dict_to_xml(data)
+        else:
+            converted_data = str(data)
+    elif content_type == ContentType.HTML.value or content_type == 'text/html':
+        if isinstance(data, dict):
+            converted_data = json.dumps(data)
+        else:
+            converted_data = str(data)
+    else:
+        if isinstance(data, dict):
+            converted_data = json.dumps(data)
+        else:
+            converted_data = str(data)
 
     return converted_data, files
 
+def dict_to_xml(data, root_tag='root'):
+    def build_xml(d, tag='item'):
+        if isinstance(d, dict):
+            xml_parts = [f'<{tag}>']
+            for key, value in d.items():
+                xml_parts.append(build_xml(value, key))
+            xml_parts.append(f'</{tag}>')
+            return ''.join(xml_parts)
+        elif isinstance(d, list):
+            return ''.join([build_xml(item, tag) for item in d])
+        else:
+            return f'<{tag}>{d}</{tag}>'
+
+    xml_parts = [f'<{root_tag}>']
+    for key, value in data.items():
+        xml_parts.append(build_xml(value, key))
+    xml_parts.append(f'</{root_tag}>')
+    return ''.join(xml_parts)
+
 
 def get_metrics():
     sdk_name_version = SdkPrefix.SKYFLOW_PYTHON + SDK_VERSION
@@ -347,39 +399,50 @@ def parse_invoke_connection_response(api_response: requests.Response):
     content = api_response.content
     if isinstance(content, bytes):
         content = content.decode(EncodingType.UTF_8)
+
     try:
         api_response.raise_for_status()
-        try:
-            data = json.loads(content)
-            metadata = {}
-            if HttpHeader.X_REQUEST_ID in api_response.headers:
-                metadata[ResponseField.REQUEST_ID] = api_response.headers[HttpHeader.X_REQUEST_ID]
+
+        content_type = api_response.headers.get(HttpHeader.CONTENT_TYPE_LOWERCASE, '').lower()
+
+        if ContentTypeConstants.APPLICATION_JSON in content_type or not content_type:
+            try:
+                data = json.loads(content)
+            except json.JSONDecodeError:
+                data = content
+        else:
+            data = content
+
+        metadata = {}
+        if HttpHeader.X_REQUEST_ID in api_response.headers:
+            metadata[ResponseField.REQUEST_ID] = api_response.headers[HttpHeader.X_REQUEST_ID]
 
-            return InvokeConnectionResponse(data=data, metadata=metadata, errors=None)
-        except Exception as e:
-            raise SkyflowError(SkyflowMessages.Error.RESPONSE_NOT_JSON.value.format(content), status_code)
+        return InvokeConnectionResponse(data=data, metadata=metadata, errors=None)
+
     except HTTPError:
         message = SkyflowMessages.Error.API_ERROR.value.format(status_code)  
+        request_id = api_response.headers.get(HttpHeader.X_REQUEST_ID)
+
         try:
-            error_response = json.loads(content)                    
-            request_id = api_response.headers[HttpHeader.X_REQUEST_ID]
+            error_response = json.loads(content)
             error_from_client = api_response.headers.get(HttpHeader.ERROR_FROM_CLIENT)
 
-            status_code = error_response.get(ResponseField.ERROR, {}).get(ResponseField.HTTP_CODE, 500)  # Default to 500 if not found
+            status_code = error_response.get(ResponseField.ERROR, {}).get(ResponseField.HTTP_CODE, status_code)
             http_status = error_response.get(ResponseField.ERROR, {}).get(ResponseField.HTTP_STATUS)
             grpc_code = error_response.get(ResponseField.ERROR, {}).get(ResponseField.GRPC_CODE)
             details = error_response.get(ResponseField.ERROR, {}).get(ResponseField.DETAILS)
             message = error_response.get(ResponseField.ERROR, {}).get(ResponseField.MESSAGE, SkyflowMessages.Error.UNKNOWN_ERROR_DEFAULT_MESSAGE.value)
 
             if error_from_client is not None:
-                if details is None: details = []
+                if details is None: 
+                    details = []
                 error_from_client_bool = error_from_client.lower() == BooleanString.TRUE
                 details.append({ResponseField.ERROR_FROM_CLIENT: error_from_client_bool})
 
             raise SkyflowError(message, status_code, request_id, grpc_code, http_status, details)
+
         except json.JSONDecodeError:
-            message = SkyflowMessages.Error.RESPONSE_NOT_JSON.value.format(content)
-        raise SkyflowError(message, status_code)
+            raise SkyflowError(content if content else message, status_code, request_id)
 
 def parse_deidentify_text_response(api_response: DeidentifyStringResponse):
     entities = [convert_detected_entity_to_entity_info(entity) for entity in api_response.entities]
@@ -397,9 +460,15 @@ def log_and_reject_error(description, status_code, request_id, http_status=None,
     raise SkyflowError(description, status_code, request_id, grpc_code, http_status, details)
 
 def handle_exception(error, logger):
-    # handle invalid cluster ID error scenario
-    if (isinstance(error, httpx.ConnectError)):
-        handle_generic_error(error, None, SkyflowMessages.ErrorCodes.INVALID_INPUT.value, logger)
+    if isinstance(error, httpx.ConnectError):
+        description = SkyflowMessages.Error.GENERIC_API_ERROR.value
+        log_and_reject_error(description, SkyflowMessages.ErrorCodes.INVALID_INPUT.value, None, logger=logger)
+        return
+
+    if not hasattr(error, 'headers') or not hasattr(error, 'body') or error.headers is None or error.body is None:
+        description = str(error) if error else SkyflowMessages.Error.GENERIC_API_ERROR.value
+        log_and_reject_error(description, SkyflowMessages.ErrorCodes.SERVER_ERROR.value, None, logger=logger)
+        return
 
     request_id = error.headers.get(HttpHeader.X_REQUEST_ID, ErrorDefaults.UNKNOWN_REQUEST_ID)
     content_type = error.headers.get(HttpHeader.CONTENT_TYPE_LOWERCASE)
@@ -411,9 +480,9 @@ def handle_exception(error, logger):
         elif ContentTypeConstants.TEXT_PLAIN in content_type:
             handle_text_error(error, data, request_id, logger)
         else:
-            handle_generic_error(error, request_id, logger)
+            handle_generic_error_with_status(error, request_id, error.status, logger)
     else:
-        handle_generic_error(error, request_id, logger)
+        handle_generic_error_with_status(error, request_id, error.status, logger)
 
 def handle_json_error(err, data, request_id, logger):
     try:
@@ -436,12 +505,9 @@ def handle_json_error(err, data, request_id, logger):
 def handle_text_error(err, data, request_id, logger):
     log_and_reject_error(data, err.status, request_id, logger =  logger)
 
-def handle_generic_error(err, request_id, logger):
-    handle_generic_error(err, request_id, err.status, logger = logger)
-
-def handle_generic_error(err, request_id, status, logger):
+def handle_generic_error_with_status(err, request_id, status, logger):
     description = SkyflowMessages.Error.GENERIC_API_ERROR.value
-    log_and_reject_error(description, status, request_id, logger = logger)
+    log_and_reject_error(description, status, request_id, logger=logger)
 
 def encode_column_values(get_request):
     encoded_column_values = list()

skyflow/utils/constants.py

@@ -35,6 +35,8 @@ class DetectStatus:
     FAILED = 'FAILED'
     UNKNOWN = 'UNKNOWN'
 
+class Detect:
+    WAIT_TIME = 64
 
 class FileExtension:
     JSON = 'json'

skyflow/utils/enums/content_types.py

@@ -5,4 +5,5 @@ class ContentType(Enum):
     PLAINTEXT = 'text/plain'
     XML = 'text/xml'
     URLENCODED = 'application/x-www-form-urlencoded'
-    FORMDATA = 'multipart/form-data'
+    FORMDATA = 'multipart/form-data'
+    HTML = 'text/html'

skyflow/utils/validations/_validations.py

@@ -9,7 +9,7 @@
 from skyflow.utils.constants import (
     ApiKey, ResponseField, RequestParameter,
     FileUploadField,
-    DeidentifyFileRequestField, RequestOperation, ConfigType, SqlCommand, ConfigField, OptionField, CredentialField
+    DeidentifyFileRequestField, RequestOperation, ConfigType, SqlCommand, ConfigField, OptionField, CredentialField, Detect
 )
 from skyflow.utils.logger import log_info, log_error_log
 from skyflow.vault.detect import DeidentifyTextRequest, ReidentifyTextRequest, TokenFormat, Transformations, \
@@ -142,8 +142,8 @@ def validate_credentials(logger, credentials, config_id_type=None, config_id=Non
         )
         if is_expired(credentials.get(CredentialField.TOKEN), logger):
             raise SkyflowError(
-                SkyflowMessages.Error.INVALID_CREDENTIALS_TOKEN.value.format(config_id_type, config_id)
-                if config_id_type and config_id else SkyflowMessages.Error.INVALID_CREDENTIALS_TOKEN.value,
+                SkyflowMessages.Error.EXPIRED_BEARER_TOKEN.value
+                if config_id_type and config_id else SkyflowMessages.Error.EXPIRED_BEARER_TOKEN.value,
                 invalid_input_error_code
             )
     elif CredentialField.API_KEY in credentials:
@@ -247,10 +247,8 @@ def validate_connection_config(logger, config):
         SkyflowMessages.Error.INVALID_CONNECTION_URL.value.format(connection_id)
     )
 
-    if ConfigField.CREDENTIALS not in config:
-        raise SkyflowError(SkyflowMessages.Error.EMPTY_CREDENTIALS.value.format(ConfigType.CONNECTION, connection_id), invalid_input_error_code)
-
-    validate_credentials(logger, config.get(ConfigField.CREDENTIALS), ConfigType.CONNECTION, connection_id)
+    if "credentials" in config:
+            validate_credentials(logger, config.get("credentials"), "connection", connection_id)
 
     return True
 
@@ -408,7 +406,7 @@ def validate_deidentify_file_request(logger, request: DeidentifyFileRequest):
     if hasattr(request, DeidentifyFileRequestField.WAIT_TIME) and request.wait_time is not None:
         if not isinstance(request.wait_time, (int, float)):
             raise SkyflowError(SkyflowMessages.Error.INVALID_WAIT_TIME.value, invalid_input_error_code)
-        if request.wait_time < 0 and request.wait_time > 64: # noqa: PLR2004
+        if request.wait_time < 0 or request.wait_time > Detect.WAIT_TIME:
             raise SkyflowError(SkyflowMessages.Error.WAIT_TIME_GREATER_THEN_64.value, invalid_input_error_code)
 
 def validate_insert_request(logger, request):
@@ -432,9 +430,6 @@ def validate_insert_request(logger, request):
             if key is None or key == "":
                 log_error_log(SkyflowMessages.ErrorLogs.EMPTY_OR_NULL_KEY_IN_VALUES.value.format(RequestOperation.INSERT), logger = logger)
 
-            if value is None or value == "":
-                log_error_log(SkyflowMessages.ErrorLogs.EMPTY_OR_NULL_VALUE_IN_VALUES.value.format(RequestOperation.INSERT, key), logger = logger)
-
     if request.upsert is not None and (not isinstance(request.upsert, str) or not request.upsert.strip()):
         log_error_log(SkyflowMessages.ErrorLogs.EMPTY_UPSERT.value(RequestOperation.INSERT), logger = logger)
         raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_OPTIONS_TYPE.value, invalid_input_error_code)
@@ -592,8 +587,8 @@ def validate_get_request(logger, request):
         raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_VALUE.value.format(type(column_values)), invalid_input_error_code)
 
     if column_name and not column_values:
-        log_error_log(SkyflowMessages.ErrorLogs.COLUMN_NAME_IS_REQUIRED.value.format(RequestOperation.GET), logger = logger)
-        SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code)
+        log_error_log(SkyflowMessages.ErrorLogs.COLUMN_NAME_IS_REQUIRED.value.format("GET"), logger = logger)
+        raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_VALUES.value, invalid_input_error_code)
 
     if (column_name or column_values) and skyflow_ids:
         log_error_log(SkyflowMessages.ErrorLogs.BOTH_IDS_AND_COLUMN_NAME_PASSED.value.format(RequestOperation.GET), logger = logger)