From 8fa6610bd925e72ba82317ae3711c40de38fdd8c Mon Sep 17 00:00:00 2001 From: Sallsamba Date: Wed, 11 Jun 2025 08:19:29 +0200 Subject: [PATCH 1/6] fix bug in xport format handling in extractor --- CHANGELOG.md | 4 ++++ README.md | 2 +- plugin.json | 2 +- python-formats/sas/format.py | 37 ++++++++++++++++++++++++++++++++++-- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb69455..b70eff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Version 1.5.0 - Bugfix release - 2025-06-11 + +- Fix .xport format handling + ## Version 1.4.0 - Bugfix release - 2025-02-18 - Improved compatibility by using more recent Pandas version diff --git a/README.md b/README.md index 7274b29..538c9f2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Plugin information -This plugin provides a custom format reader to import the contents of a .sas7bdat data file into your DSS project. +This plugin provides a custom format reader to import the contents of a .sas7bdat or .xport data file into your DSS project. This implementation will work with almost any SAS file but may be slower than the default one provided with DSS. It is provided by the Pandas library shipped with DSS, and may change when future Pandas versions are released. diff --git a/plugin.json b/plugin.json index 4b2ca19..cd16334 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "sas-format", - "version": "1.4.0", + "version": "1.5.0", "meta": { "label": "SAS format provided by Pandas", "description": "Format supporting SAS files (.sas7bdat and .xport). This implementation is provided by Pandas and sas7bdat and may be slower than the light one provided with DSS, but supports more options.", diff --git a/python-formats/sas/format.py b/python-formats/sas/format.py index e02179c..58a1595 100644 --- a/python-formats/sas/format.py +++ b/python-formats/sas/format.py @@ -83,12 +83,19 @@ def __init__(self, stream, schema, config): encoding = config.get("encoding", "latin_1") dump_to_file = config.get("dump_to_file", False) + self.sas_format = sas_format self.hasSchema = schema != None + """necessary to handle the case where the stream is not seekable => force dump_to_file""" + if sas_format.lower() == 'xport' and not dump_to_file: + print("Warning: XPORT format detected, forcing dump_to_file mode for better compatibility") + dump_to_file = True + if dump_to_file: dirname, _ = os.path.split(os.path.abspath(__file__)) with TmpFolder(dirname) as tmp_folder_path: - fullpath = os.path.join(tmp_folder_path, 'dumped-%s.sas7bdat' % (time.time())) + extension = 'xpt' if sas_format.lower() == 'xport' else 'sas7bdat' + fullpath = os.path.join(tmp_folder_path, 'dumped-%s.%s' % (time.time(), extension)) with open(fullpath, 'wb') as of: for data in iter((lambda: stream.read(500000)), b''): of.write(data) @@ -119,7 +126,33 @@ def get_chunk(self): self.chunk_nb = 0 def read_schema(self): - return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] + # For XPORT format, we need to extract the schema from the fields + if self.sas_format.lower() == 'xport': + if hasattr(self.iterator, 'fields') and self.iterator.fields: + schema = [] + for field in self.iterator.fields: + if isinstance(field, dict): + field_name = field.get('name', 'unknown') + field_type = "DOUBLE" if field.get('ntype') == 1 else "STRING" + else: + field_name = getattr(field, 'name', 'unknown') + field_type = "DOUBLE" if getattr(field, 'ntype', 2) == 1 else "STRING" + + schema.append({"name": field_name, "type": field_type}) + return schema + else: + if hasattr(self.iterator, 'columns'): + schema = [] + for col_name in self.iterator.columns: + schema.append({"name": col_name, "type": "STRING"}) + return schema + else: + # For SAS7BDAT + if hasattr(self.iterator, 'columns') and len(self.iterator.columns) > 0: + if hasattr(self.iterator.columns[0], 'name'): + return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] + + return [] def read_row(self): try: From 7fc90b2b62ac8c0e3ba026a33857090d89cb51e9 Mon Sep 17 00:00:00 2001 From: Sallsamba Date: Wed, 11 Jun 2025 08:27:31 +0200 Subject: [PATCH 2/6] add comments for more clarity --- python-formats/sas/format.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python-formats/sas/format.py b/python-formats/sas/format.py index 58a1595..2bb3d97 100644 --- a/python-formats/sas/format.py +++ b/python-formats/sas/format.py @@ -86,7 +86,7 @@ def __init__(self, stream, schema, config): self.sas_format = sas_format self.hasSchema = schema != None - """necessary to handle the case where the stream is not seekable => force dump_to_file""" + # necessary to handle the case where the stream is not seekable => force dump_to_file if sas_format.lower() == 'xport' and not dump_to_file: print("Warning: XPORT format detected, forcing dump_to_file mode for better compatibility") dump_to_file = True @@ -128,6 +128,7 @@ def get_chunk(self): def read_schema(self): # For XPORT format, we need to extract the schema from the fields if self.sas_format.lower() == 'xport': + # checking if the iterator has fields or columns if hasattr(self.iterator, 'fields') and self.iterator.fields: schema = [] for field in self.iterator.fields: @@ -148,6 +149,7 @@ def read_schema(self): return schema else: # For SAS7BDAT + # checking if the iterator has columns if hasattr(self.iterator, 'columns') and len(self.iterator.columns) > 0: if hasattr(self.iterator.columns[0], 'name'): return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] From d725d0e52159111a8dac3ae6a70d0760a046494d Mon Sep 17 00:00:00 2001 From: Sallsamba Date: Wed, 11 Jun 2025 08:33:11 +0200 Subject: [PATCH 3/6] simplify read_schema logic --- python-formats/sas/format.py | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/python-formats/sas/format.py b/python-formats/sas/format.py index 2bb3d97..31e43b2 100644 --- a/python-formats/sas/format.py +++ b/python-formats/sas/format.py @@ -126,35 +126,10 @@ def get_chunk(self): self.chunk_nb = 0 def read_schema(self): - # For XPORT format, we need to extract the schema from the fields - if self.sas_format.lower() == 'xport': - # checking if the iterator has fields or columns - if hasattr(self.iterator, 'fields') and self.iterator.fields: - schema = [] - for field in self.iterator.fields: - if isinstance(field, dict): - field_name = field.get('name', 'unknown') - field_type = "DOUBLE" if field.get('ntype') == 1 else "STRING" - else: - field_name = getattr(field, 'name', 'unknown') - field_type = "DOUBLE" if getattr(field, 'ntype', 2) == 1 else "STRING" - - schema.append({"name": field_name, "type": field_type}) - return schema - else: - if hasattr(self.iterator, 'columns'): - schema = [] - for col_name in self.iterator.columns: - schema.append({"name": col_name, "type": "STRING"}) - return schema - else: - # For SAS7BDAT - # checking if the iterator has columns - if hasattr(self.iterator, 'columns') and len(self.iterator.columns) > 0: - if hasattr(self.iterator.columns[0], 'name'): - return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] - - return [] + if hasattr(self.iterator, 'fields'): # XPORT format + return [{"name": f.name, "type": "DOUBLE" if f.ntype == 'numeric' else "STRING"} for f in self.iterator.fields] + else: # SAS7BDAT format + return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] def read_row(self): try: From bedaa86aedf3091a8df650311ae2dc6f471c6390 Mon Sep 17 00:00:00 2001 From: Sallsamba Date: Wed, 11 Jun 2025 08:38:46 +0200 Subject: [PATCH 4/6] cleaning --- python-formats/sas/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python-formats/sas/format.py b/python-formats/sas/format.py index 31e43b2..2126ed0 100644 --- a/python-formats/sas/format.py +++ b/python-formats/sas/format.py @@ -83,7 +83,6 @@ def __init__(self, stream, schema, config): encoding = config.get("encoding", "latin_1") dump_to_file = config.get("dump_to_file", False) - self.sas_format = sas_format self.hasSchema = schema != None # necessary to handle the case where the stream is not seekable => force dump_to_file From b1116dc557c00664914d44076c9cff79a066351f Mon Sep 17 00:00:00 2001 From: Sallsamba Date: Wed, 11 Jun 2025 08:48:56 +0200 Subject: [PATCH 5/6] revert unworking simplification --- python-formats/sas/format.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/python-formats/sas/format.py b/python-formats/sas/format.py index 2126ed0..2bb3d97 100644 --- a/python-formats/sas/format.py +++ b/python-formats/sas/format.py @@ -83,6 +83,7 @@ def __init__(self, stream, schema, config): encoding = config.get("encoding", "latin_1") dump_to_file = config.get("dump_to_file", False) + self.sas_format = sas_format self.hasSchema = schema != None # necessary to handle the case where the stream is not seekable => force dump_to_file @@ -125,10 +126,35 @@ def get_chunk(self): self.chunk_nb = 0 def read_schema(self): - if hasattr(self.iterator, 'fields'): # XPORT format - return [{"name": f.name, "type": "DOUBLE" if f.ntype == 'numeric' else "STRING"} for f in self.iterator.fields] - else: # SAS7BDAT format - return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] + # For XPORT format, we need to extract the schema from the fields + if self.sas_format.lower() == 'xport': + # checking if the iterator has fields or columns + if hasattr(self.iterator, 'fields') and self.iterator.fields: + schema = [] + for field in self.iterator.fields: + if isinstance(field, dict): + field_name = field.get('name', 'unknown') + field_type = "DOUBLE" if field.get('ntype') == 1 else "STRING" + else: + field_name = getattr(field, 'name', 'unknown') + field_type = "DOUBLE" if getattr(field, 'ntype', 2) == 1 else "STRING" + + schema.append({"name": field_name, "type": field_type}) + return schema + else: + if hasattr(self.iterator, 'columns'): + schema = [] + for col_name in self.iterator.columns: + schema.append({"name": col_name, "type": "STRING"}) + return schema + else: + # For SAS7BDAT + # checking if the iterator has columns + if hasattr(self.iterator, 'columns') and len(self.iterator.columns) > 0: + if hasattr(self.iterator.columns[0], 'name'): + return [{"name": c.name, "type": "DOUBLE" if c.ctype == 'd' else "STRING"} for c in self.iterator.columns] + + return [] def read_row(self): try: From 88b65822a03c63e75c74ea1281aea0f3b2affc12 Mon Sep 17 00:00:00 2001 From: Sallsamba Date: Wed, 11 Jun 2025 14:29:46 +0200 Subject: [PATCH 6/6] use in-memory buffer when dump_to_file is false --- python-formats/sas/format.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python-formats/sas/format.py b/python-formats/sas/format.py index 2bb3d97..5f6ae92 100644 --- a/python-formats/sas/format.py +++ b/python-formats/sas/format.py @@ -86,12 +86,18 @@ def __init__(self, stream, schema, config): self.sas_format = sas_format self.hasSchema = schema != None - # necessary to handle the case where the stream is not seekable => force dump_to_file if sas_format.lower() == 'xport' and not dump_to_file: - print("Warning: XPORT format detected, forcing dump_to_file mode for better compatibility") - dump_to_file = True - - if dump_to_file: + buffer = io.BytesIO() + for data in iter((lambda: stream.read(500000)), b''): + buffer.write(data) + buffer.seek(0) + + self.iterator = pd.read_sas(buffer, + format=sas_format, + iterator=True, + encoding=encoding, + chunksize=chunksize) + elif dump_to_file: dirname, _ = os.path.split(os.path.abspath(__file__)) with TmpFolder(dirname) as tmp_folder_path: extension = 'xpt' if sas_format.lower() == 'xport' else 'sas7bdat'