diff --git a/pyproject.toml b/pyproject.toml index 1b196ee..f3057d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hec-dss-python" -version = "0.1.28" +version = "0.1.29" description = "Python wrapper for the HEC-DSS file database C library." authors = ["Hydrologic Engineering Center"] license = "MIT" diff --git a/setup.cfg b/setup.cfg index 38066a4..0e81e2c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = hecdss -version = 0.1.28 +version = 0.1.29 author = Hydrologic Engineering Center author_email =hec.dss@usace.army.mil description = Python wrapper for the HEC-DSS file database C library. diff --git a/src/hecdss/download_hecdss.py b/src/hecdss/download_hecdss.py index ad07c1b..0ecc0d7 100644 --- a/src/hecdss/download_hecdss.py +++ b/src/hecdss/download_hecdss.py @@ -36,7 +36,7 @@ def download_and_unzip(url, zip_file, destination_dir): print(f"Failed to download zip file. Status code: {response.status_code}") base_url = "https://www.hec.usace.army.mil/nexus/repository/maven-public/mil/army/usace/hec/hecdss/" -version = "7-IW-4" +version = "7-JA-4" destination_dir = Path(__file__).parent.joinpath("lib") zip_url = f"{base_url}{version}-win-x86_64/hecdss-{version}-win-x86_64.zip" diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index de50b43..51df75c 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -679,6 +679,24 @@ def put(self, container) -> int: return status + + def writePrecompressedGrid(self, gd, compressedData, CompressionSize): + """ + puts pre-compressed gridded data into the DSS file + + Args + compressedData (bytes): Compressed data. + CompressionSize (int): Size of the compressed data. + Returns: + int: 0 if successful, -1 otherwise. + """ + + if compressedData and CompressionSize > 0: + status = self._native.hec_dss_gridStore(gd, compressedData, CompressionSize) + self._catalog = None + return status + return -1 + def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, enddatetime=None) -> int: """deletes a record from the DSS file Args: diff --git a/src/hecdss/native.py b/src/hecdss/native.py index bbc8149..a85a8b9 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -332,6 +332,8 @@ def hec_dss_gridRetrieve(self, pathname: str, def hec_dss_gridStore( self, gd, + compressedData=None, + compressionSize=0, ): self.dll.hec_dss_pdStore.restype = c_int self.dll.hec_dss_pdStore.argtypes = [ @@ -348,6 +350,7 @@ def hec_dss_gridStore( ctypes.c_int, # timeZoneRawOffset ctypes.c_int, # isInterval ctypes.c_int, # isTimeStamped + ctypes.c_int, # compressionSize ctypes.c_char_p, # dataUnits ctypes.c_char_p, # dataSource ctypes.c_char_p, # srsName @@ -378,6 +381,7 @@ def hec_dss_gridStore( c_timeZoneRawOffset = c_int(gd.timeZoneRawOffset) c_isInterval = c_int(gd.isInterval) c_isTimeStamped = c_int(gd.isTimeStamped) + c_compressionSize = c_int(compressionSize) # default compression c_dataUnits = c_char_p(gd.dataUnits.encode("utf-8")) c_dataSource = c_char_p(gd.dataSource.encode("utf-8")) @@ -397,15 +401,19 @@ def hec_dss_gridStore( c_numberEqualOrExceedingRangeLimit = (c_int * len(gd.numberEqualOrExceedingRangeLimit))( *gd.numberEqualOrExceedingRangeLimit) - arr = gd.data.astype('float32', copy=False) - c_data = arr.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) + if compressedData is not None and compressionSize: + # Treat compressed data as raw bytes, not float32 + c_data = ctypes.cast(compressedData, ctypes.POINTER(ctypes.c_float)) + else: + arr = gd.data.astype('float32', copy=False) + c_data = arr.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) return self.dll.hec_dss_gridStore(self.handle, c_pathname, c_gridType, c_dataType, c_lowerLeftCellX, c_lowerLeftCellY, c_numberOfCellsX, c_numberOfCellsY, c_numberOfRanges, c_srsDefinitionType, c_timeZoneRawOffset, c_isInterval, - c_isTimeStamped, + c_isTimeStamped, c_compressionSize, c_dataUnits, c_dataSource, c_srsName, c_srsDefinition, c_timeZoneID, c_cellSize, c_xCoordOfGridCellZero, diff --git a/tests/test_gridded_data.py b/tests/test_gridded_data.py index 2ecb94b..f88ae8f 100644 --- a/tests/test_gridded_data.py +++ b/tests/test_gridded_data.py @@ -86,6 +86,44 @@ def test_gridded_data_read_store_read(self): assert ( gd.dataUnits == gd2.dataUnits), f"gd2.dataUnits is not equal to {gd.dataUnits}. gd2.dataUnits is {gd2.dataUnits}" + def test_gridded_data_write_precompressed(self): + """ + Test writing precompressed gridded data using zlib deflate. + Reads existing grid data, compresses it, writes using writePrecompressedGrid, and compares. + """ + import zlib + + # Read existing gridded data + original_path = "/grid/EAU GALLA RIVER/SNOW MELT/02FEB2020:0600/03FEB2020:0600/SHG-SNODAS/" + file = self.test_files.get_copy("grid-example.dss") + + with HecDss(file) as dss: + # Read the original grid + gd_original = dss.get(original_path) + + # Compress the data using zlib deflate + raw_bytes = gd_original.data.astype(np.float32).tobytes() + compressed_data = zlib.compress(raw_bytes) + compression_size = len(compressed_data) + + # Create a new GriddedData object with metadata from original + # but pointing to a new path + new_path = "/grid/EAU GALLA RIVER/SNOW MELT/02FEB2020:0600/03FEB2020:0600/SHG-SNODAS-COMPRESSED/" + gd_original.id = new_path + + + # Write the precompressed grid + status = dss.writePrecompressedGrid(gd_original, compressed_data, compression_size) + + # Read back the compressed grid + gd_readback = dss.get(new_path) + + # Compare the two grids + assert status == 0, f"writePrecompressedGrid status should be 0, is {status}" + assert np.array_equal(gd_original.data, gd_readback.data), "Data from original and compressed grid do not match" + assert gd_original.numberOfCellsX == gd_readback.numberOfCellsX, "numberOfCellsX mismatch" + assert gd_original.numberOfCellsY == gd_readback.numberOfCellsY, "numberOfCellsY mismatch" + assert gd_original.dataUnits == gd_readback.dataUnits, "dataUnits mismatch" if __name__ == "__main__":