from obstore.store import from_url
from virtualizarr import open_virtual_dataset
from virtualizarr.parsers import KerchunkJSONParser, HDFParser, NetCDF3Parser
from obspec_utils.registry import ObjectStoreRegistry
import xarray as xr
import json
if __name__ == '__main__':
final = 'test.json'
infiles = [
"/badc/cmip6/data/CMIP6/ScenarioMIP/MOHC/UKESM1-0-LL/ssp370/r10i1p1f2/AERday/maxpblz/gn/v20200603/maxpblz_AERday_UKESM1-0-LL_ssp370_r10i1p1f2_gn_20150101-20491230.nc",
"/badc/cmip6/data/CMIP6/ScenarioMIP/MOHC/UKESM1-0-LL/ssp370/r10i1p1f2/AERday/maxpblz/gn/v20200603/maxpblz_AERday_UKESM1-0-LL_ssp370_r10i1p1f2_gn_20500101-21001230.nc"
]
parser = HDFParser()
store = from_url("file://")
registry = ObjectStoreRegistry({"file://": store})
registry.register(f'file://{infiles[0]}', store)
files = []
for x, f in enumerate(infiles):
vds = open_virtual_dataset(f, parser=parser, registry=registry)
vds.virtualize.to_kerchunk(f"{x}.json", format='json')
files.append(f'{x}.json')
file_path = files[0]
file_url = f"file:///home/users/dwest77/cedadev/padocc/vcache/{file_path}"
store = from_url("file://")
registry = ObjectStoreRegistry({"file://": store})
registry.register(file_url, store)
parser = KerchunkJSONParser()
vds = []
for f in files:
vds.append(open_virtual_dataset(f, parser=parser, registry=registry))
combined_vds = xr.combine_nested(vds, concat_dim=['time'], coords='minimal',compat='override', combine_attrs='override')
combined_vds.virtualize.to_kerchunk(final, format='json')
Using open_virtual_dataset to parse a KerchunkJSON object with a fill_value
nullresults inTypeError: Failed to decode fill_value: expected str or bytes for dtype float64, got float. This is because in the xarray-zarr backendFillValueCoderclass on decoding a fill value, for np_dtype.kind == 'f' the value must be str or bytes, except in my case it's 'null'.The file source is (download locally)
Example code to replicate this issue: