From 5be2757c1c00af42a760d460413dd80b54b39450 Mon Sep 17 00:00:00 2001 From: kristiker Date: Wed, 22 Mar 2023 16:42:21 +0100 Subject: [PATCH 1/9] Use keyvalues3 library --- requirements.txt | 3 ++ utils/shared/keyvalues3.py | 98 -------------------------------------- 2 files changed, 3 insertions(+), 98 deletions(-) delete mode 100644 utils/shared/keyvalues3.py diff --git a/requirements.txt b/requirements.txt index 3ac5998..3bd23a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,9 @@ Pillow numpy +# Particles, prop_data +keyvalues3==0.1a1 + # Maps vdf dataclassy==0.10.4 diff --git a/utils/shared/keyvalues3.py b/utils/shared/keyvalues3.py deleted file mode 100644 index 04e8a41..0000000 --- a/utils/shared/keyvalues3.py +++ /dev/null @@ -1,98 +0,0 @@ -from pathlib import Path -from dataclasses import dataclass -from uuid import UUID - -@dataclass(frozen=True) -class KV3Header: - encoding: str = 'text' - encoding_ver: str = 'e21c7f3c-8a33-41c5-9977-a76d3a32aa0d' - format: str = 'generic' - format_ver: str = '7412167c-06e9-4698-aff2-e63eb59037e7' - _common = '' - def __str__(self): - return self._common % (self.encoding, self.encoding_ver, self.format, self.format_ver) - -@dataclass(frozen=True) -class resource: - path: Path - def __str__(self): - return f'resource:"{self.path.as_posix().lower()}"' - -class KV3File(dict): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.header = KV3Header(format="source1imported") - - def __str__(self): - kv3 = str(self.header) + '\n' - - def obj_serialize(obj, indent = 1, dictKey = False): - preind = ('\t' * (indent-1)) - ind = ('\t' * indent) - if obj is None: - return 'null' - elif isinstance(obj, bool): - if obj: return 'true' - return 'false' - elif isinstance(obj, str): - return '"' + obj + '"' - elif isinstance(obj, list): - s = '[' - if any(isinstance(item, dict) for item in obj): # TODO: only non numbers - s = f'\n{preind}[\n' - for item in obj: - s += (obj_serialize(item, indent+1) + ',\n') - return s + preind + ']\n' - - return f'[{", ".join((obj_serialize(item, indent+1) for item in obj))}]' - elif isinstance(obj, dict): - s = preind + '{\n' - if dictKey: - s = '\n' + s - for key, value in obj.items(): - #if value == [] or value == "" or value == {}: continue - if not isinstance(key, str): - key = f'"{key}"' - s += ind + f"{key} = {obj_serialize(value, indent+1, dictKey=True)}\n" - return s + preind + '}' - else: # int, float, resource - # round off inaccurate dmx floats - if type(obj) == float: - obj = round(obj, 6) - return str(obj) - - kv3 += obj_serialize(self) - - return kv3 - - def ToString(self): - return self.__str__() - -if __name__ == '__main__': - import unittest - class Test_KV3(unittest.TestCase): - default_header = '' - default_s1import_context_header = '' - def test_default_header(self): - self.assertEqual(str(KV3Header()), self.default_header) - - def test_custom_header(self): - header = KV3Header('text2', '123-123', 'generic2', '234-234') - headertext = '' - self.assertEqual(str(header), headertext) - - def test_kv(self): - expect_text = f'{self.default_s1import_context_header}\n{{'+\ - '\n\ta = "asd asd"'+\ - '\n\tb = \n\t{\n\t\t"(2,)" = 3\n\t}'+\ - '\n\tc = ["listed_text1", "listed_text2"]\n}' - self.assertEqual( - KV3File( - a='asd asd', - b={(2,):3}, - c=["listed_text1", "listed_text2"] - ).ToString(), - expect_text - ) - - unittest.main() From af390160530d9888ad3bd934de00f09bcfe4e5ff Mon Sep 17 00:00:00 2001 From: kristiker Date: Wed, 22 Mar 2023 16:47:58 +0100 Subject: [PATCH 2/9] fix build --- .github/workflows/unittests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 5320750..94a9676 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -23,10 +23,10 @@ jobs: with: submodules: true - - name: Setup Python 3.10 + - name: Setup Python 3.101 uses: actions/setup-python@v4.2.0 with: - python-version: 3.10.6 + python-version: 3.11 # Runs a single command using the runners shell - name: Dependencies @@ -39,7 +39,6 @@ jobs: python utils/shared/base_utils2.py -i "$PWD" -e "$PWD" python utils/shared/cstr.py python utils/shared/cppkeyvalues.py - python utils/shared/keyvalues3.py python utils/shared/material_proxies.py python utils/shared/qc.py From eb9a4e8fce4c275064a11494d33d20f2e766ef15 Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:30:30 +0200 Subject: [PATCH 3/9] shared.kv3 -> kv3 --- utils/models_import.py | 55 ++++++++++----------------------------- utils/particles_import.py | 17 ++++++------ utils/shared/modeldoc.py | 35 ++++++++++++++++++++----- 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/utils/models_import.py b/utils/models_import.py index 771f2b1..1f9e2a3 100644 --- a/utils/models_import.py +++ b/utils/models_import.py @@ -2,10 +2,10 @@ import shutil from typing import Literal, Type, Union import shared.base_utils2 as sh +import kv3 from pathlib import Path from itertools import tee from srctools import smd -from shared.keyvalues3 import KV3File, KV3Header """ Import Source Engine models to Source 2 @@ -93,11 +93,10 @@ def main(): def ImportMDLtoVMDL(mdl_path: Path): - vmdl_path = mdl_path.with_suffix('.vmdl') - vmdl = KV3File( + vmdl = dict( m_sMDLFilename = ("../"*SAMPBOX) + mdl_path.local.as_posix() ) - vmdl_path.write_text(vmdl.ToString()) + kv3.write(vmdl, mdl_path.with_suffix('.vmdl')) print('+ Generated', vmdl_path.local) return vmdl_path @@ -117,7 +116,7 @@ def ImportMDLtoVMDL(mdl_path: Path): def ImportQCtoVMDL(qc_path: Path): - vmdl = ModelDocVMDL() + vmdl = ModelDoc() # local paths active_folder: Path = qc_path.local.parent @@ -192,8 +191,8 @@ def add_rendermesh_from_body(body: QC.body): for command in qc_commands: match command: case QC.staticprop(): - vmdl.root.model_archetype = "static_prop_model" - vmdl.root.primary_associated_entity = "prop_static" + vmdl.rootNode.model_archetype = "static_prop_model" + vmdl.rootNode.primary_associated_entity = "prop_static" case QC.popd(): try: active_folder = dir_stack.pop() @@ -526,7 +525,7 @@ def add_rendermesh_from_body(body: QC.body): # grab $animation, $sequence, $attachment and $collisiontext from this model elif isinstance(command, QC.includemodel): command: QC.includemodel - vmdl.root.base_model_name = (models / command.filename).with_suffix('.vmdl').as_posix() + vmdl.rootNode.base_model_name = (models / command.filename).with_suffix('.vmdl').as_posix() elif isinstance(command, QC.declaresequence): sequences_declared.append(command.name) @@ -606,7 +605,7 @@ def add_rendermesh_from_body(body: QC.body): out_vmdl_path.parent.MakeDir() if len(sequences_declared): - vmdl_prefab = ModelDocVMDL() + vmdl_prefab = ModelDoc() out_vmdl_prefab_path = out_vmdl_path.with_name("declared_sequences.vmdl_prefab") for sequence in sequences_declared: @@ -615,45 +614,19 @@ def add_rendermesh_from_body(body: QC.body): ) vmdl_prefab.add_to_appropriate_list(animfile) - out_vmdl_prefab_path.write_text(vmdl_prefab.ToString()) + kv3.write(vmdl_prefab, out_vmdl_prefab_path) print('+ Saved prefab', out_vmdl_prefab_path.local) if len(skeleton.children): - vmdl.root.add_nodes(skeleton) + vmdl.rootNode.add_nodes(skeleton) - out_vmdl_path.write_text(vmdl.ToString()) + kv3.write(vmdl, out_vmdl_path) print('+ Saved', out_vmdl_path.local) -from dataclasses import asdict -class ModelDocVMDL(KV3File): - def __init__(self): - self.header = KV3Header( - format='source1imported', - format_ver='3cec427c-1b0e-4d48-a90a-0436f33a6041' if sh.SBOX else 'fb63b6ca-f435-4aa0-a2c7-c66ddc651dca' - ) - self.root = ModelDoc.RootNode() - - self.base_lists: dict[Type[_BaseNode], _BaseNode] = {} - - def __str__(self): - self["rootNode"] = asdict(self.root) - return super().__str__() - - def add_to_appropriate_list(self, node: _Node): - """ - Adds bodygroup to bodygrouplist, animfile to animationlist, etc. Only makes one list. - """ - container_type = ModelDoc.get_container(type(node)) - container = self.base_lists.get(container_type) - if container is None: - if container_type is None: - raise RuntimeError(f"Don't know where {type(node)} belongs.") - container = container_type() - self.base_lists[container_type] = container - self.root.add_nodes(container) - - container.add_nodes(node) +from dataclasses import dataclass, asdict + + if __name__ == "__main__": # TODO: Don't ask for src1? diff --git a/utils/particles_import.py b/utils/particles_import.py index 4e46f09..f3176eb 100644 --- a/utils/particles_import.py +++ b/utils/particles_import.py @@ -1,6 +1,6 @@ import shared.base_utils2 as sh import shared.datamodel as dmx -import shared.keyvalues3 as kv3 +import keyvalues3 as kv3 from dataclasses import dataclass from pathlib import Path @@ -152,7 +152,7 @@ def __call__(self, oldval, existing = default.copy()): # }, # ] 'sequence 0 model': lambda mdl_path: ('m_ModelList', - [{'m_model': kv3.resource(Path('models/' + mdl_path).with_suffix('.vmdl'))}] + [{'m_model': kv3.flagged_value(Path('models/' + mdl_path).with_suffix('.vmdl'), kv3.Flag.resource)}] ), 'orient model z to normal': 'm_bOrientZ', 'activity override': 'm_ActivityName', @@ -1831,7 +1831,7 @@ def process_material(value: str): vmt_path = sh.IMPORT_GAME / "materials" / value # vmts are found in game (as most things) vmat_path = vmt_path.local.with_suffix('.vmat') - vpcf._base_t['m_Renderers']['m_hMaterial'] = kv3.resource(vmat_path) + vpcf._base_t['m_Renderers']['m_hMaterial'] = kv3.flagged_value(vmat_path, kv3.Flag.resource) try: vmt = VMT(KV.FromFile(vmt_path)) except FileNotFoundError: @@ -1864,7 +1864,7 @@ def process_material(value: str): with open(vtex_path, 'w') as fp: fp.write(VTEX_TEMPLATE.replace('<>', tex.as_posix(), 1)) vpcf_replacement_key = 'm_hTexture' if vmtkey in ('$basetexture', '$material') else 'm_hNormalTexture' - vpcf._base_t['m_Renderers'][vpcf_replacement_key] = kv3.resource(vtex_path.local) + vpcf._base_t['m_Renderers'][vpcf_replacement_key] = kv3.flagged_value(vtex_path.local, kv3.Flag.resource) continue if vmtkey not in vmt_to_vpcf: #un((vmtkey, vmtval), "VMT") @@ -1905,7 +1905,7 @@ def pcfkv_convert(key, value): return if key == 'snapshot': vsnaps[vpcf.path.local] = value - return str(vpcf_translation), kv3.resource(Path(vpcf.path.local.parent / (value + '.vpcf'))) + return str(vpcf_translation), kv3.flagged_value(Path(vpcf.path.local.parent / (value + '.vpcf')), kv3.Flag.resource) return vpcf_translation, value elif isinstance(vpcf_translation, tuple): @@ -1984,7 +1984,7 @@ def pcfkv_convert(key, value): if isinstance(value2, dmx.Element): value2 = value2.name else: input(f'Ref not an element {key2}: {value2}') - value2 = kv3.resource(Path(vpcf.path.local.parent / (value2 + '.vpcf'))) + value2 = kv3.flagged_value(Path(vpcf.path.local.parent / (value2 + '.vpcf')), kv3.Flag.resource) elif isinstance(subkey, (minof, maxof)): bMin = isinstance(subkey, minof) if str(subkey) in subKV: @@ -2053,10 +2053,9 @@ def ImportParticleSnapshotFile(psf_path: Path) -> Path: vsnap_path.parent.MakeDir() return copyfile(psf_path, vsnap_path) -class VPCF(kv3.KV3File): +class VPCF(dict): def __init__(self, path, **kwargs): super().__init__(**kwargs) - self.header = kv3.KV3Header(format='vpcf26', format_ver='26288658-411e-4f14-b698-2e1e5d00dec6') self['_class'] = 'CParticleSystemDefinition' self.update(kwargs) @@ -2100,7 +2099,7 @@ def ImportPSD(ParticleSystemDefinition: dmx.Element, out_root: Path, bOverwrite vpcf.setdefault('m_PreEmissionOperators', list()) vpcf['m_PreEmissionOperators'].append(operator) - vpcf.path.write_text(vpcf.ToString()) + kv3.write(dict(vpcf), vpcf.path, format=kv3.Format("vpcf26", UUID("26288658-411e-4f14-b698-2e1e5d00dec6"))) print("+ Saved", vpcf.path.local.as_posix()) diff --git a/utils/shared/modeldoc.py b/utils/shared/modeldoc.py index 34ab75a..8145fec 100644 --- a/utils/shared/modeldoc.py +++ b/utils/shared/modeldoc.py @@ -58,8 +58,9 @@ def inner(cls): return cls return inner +@dataclass class ModelDoc: - + @dataclass class RootNode(_BaseNode): model_archetype: str = "" @@ -205,12 +206,6 @@ class Bounds_View(_Node): class Prefab(_Node): target_file: resourcepath = "" - @staticmethod - def get_container(node_type: Type[_Node]): - for basecontainer in mdBaseLists: - if node_type in basecontainer._childtypes: - return basecontainer - #@containerof(BodyGroupChoice) @dataclass class BodyGroup(_Node): @@ -254,3 +249,29 @@ class ModelDataList(_BaseNode): pass @containerof(Prefab) class PrefabList(_BaseNode): pass + + rootNode: RootNode = field(default_factory=RootNode) + + @staticmethod + def get_container(node_type: Type[_Node]): + for basecontainer in mdBaseLists: + if node_type in basecontainer._childtypes: + return basecontainer + + def __post_init__(self): + self.base_lists: dict[Type[_BaseNode], _BaseNode] = field(default_factory=dict, init=False) + + def add_to_appropriate_list(self, node: _Node): + """ + Adds bodygroup to bodygrouplist, animfile to animationlist, etc. Only makes one list. + """ + container_type = ModelDoc.get_container(type(node)) + container = self.base_lists.get(container_type) + if container is None: + if container_type is None: + raise RuntimeError(f"Don't know where {type(node)} belongs.") + container = container_type() + self.base_lists[container_type] = container + self.root.add_nodes(container) + + container.add_nodes(node) From 2c1c372de0db4de2840c6a8e40412a5b36becb6a Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:37:07 +0200 Subject: [PATCH 4/9] use kv3 in scripts --- utils/scripts_import.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/utils/scripts_import.py b/utils/scripts_import.py index 79c405b..de8c4c4 100644 --- a/utils/scripts_import.py +++ b/utils/scripts_import.py @@ -2,7 +2,7 @@ from shutil import copyfile from pathlib import Path from shared.keyvalues1 import KV, VDFDict -from shared.keyvalues3 import KV3File +import keyvalues3 import itertools OVERWRITE_ASSETS = False @@ -84,7 +84,7 @@ def main(): sh.MakeDir(sh.output(scripts)) if not (sh.output(propdata, ".vdata").is_file() and not OVERWRITE_ASSETS): kv = KV.FromFile(propdata, case_sensitive=True) - kv3 = KV3File(generic_data_type = "prop_data") + kv3 = dict(generic_data_type = "prop_data") for name, data in kv.items(): if name.lower() == "breakablemodels": continue @@ -93,7 +93,7 @@ def main(): if key.lower() == "base": key = "_base" kv3[name][key] = value - sh.output(propdata, ".vdata").write_text(kv3.ToString()) + keyvalues3.write(kv3, sh.output(propdata, ".vdata")) print("+ Saved scripts/propdata.vdata") else: sh.import_context['dest'] = sh.EXPORT_GAME @@ -185,7 +185,7 @@ def ImportSoundscapesToVdata(file: Path): } } """ - sndscape_file.write_text(KV3File(data=sndscape_data).ToString()) + keyvalues3.write(dict(data=(data=sndscape_data)), sndscape_file) print("+ Saved", sndscape_file.local) return sndscape_folder @@ -277,7 +277,7 @@ def _handle_range(k, v) -> "median, deviation": return out_v, range kv = KV.CollectionFromFile(asset_path) - kv3 = KV3File() + kv3 = dict() for gamesound, gs_data in kv.items(): # "weapon.fire", {} @@ -379,7 +379,7 @@ def _handle_range(k, v) -> "median, deviation": out_kv[out_k] = out_v if sh.SBOX: - sound_file.write_text(KV3File(data=sound_data).ToString()) + keyvalues3.write(dict(data=sound_data), sound_file) print("+ Saved", sound_file.local) else: if out_kv == dict(type='src1_3d'): # empty @@ -396,7 +396,7 @@ def _handle_range(k, v) -> "median, deviation": if sh.SBOX: return out_sound_folder else: - vsndevts_file.write_text(kv3.ToString()) + keyvalues3.write(kv3, vsndevts_file) print("+ Saved", vsndevts_file.local) return vsndevts_file @@ -432,7 +432,7 @@ def ImportSurfaceProperties(asset_path: Path): surface_folder.MakeDir() surface_collection = KV.CollectionFromFile(asset_path) - vsurf = KV3File(SurfacePropertiesList = []) + vsurf = dict(SurfacePropertiesList = []) for surface, properties in {**surface_collection}.items(): new_surface = dict(surfacePropertyName = surface) @@ -486,7 +486,7 @@ def ImportSurfaceProperties(asset_path: Path): surface_data["Sounds"][key] = value if sh.SBOX: - surface_file.write_text(KV3File(data=surface_data).ToString()) + keyvalues3.write(dict(data=surface_data), surface_file) print("+ Saved", surface_file.local) else: # Add default base @@ -503,7 +503,7 @@ def ImportSurfaceProperties(asset_path: Path): if sh.SBOX: return surface_folder else: - vsurf_file.write_text(vsurf.ToString()) + keyvalues3.write(vsurf, vsurf_file) print("+ Saved", vsurf_file.local) return vsurf_file, vsurf['SurfacePropertiesList'] @@ -532,7 +532,7 @@ def after_all_converted(self): if not (self.manifest_files and self.all_surfaces): return vsurf_path = next(iter(self.all_surfaces)).with_stem('surfaceproperties') - vsurf = KV3File(SurfacePropertiesList = []) + vsurf = dict(SurfacePropertiesList = []) for file in self.manifest_files[::-1]: file = vsurf_path.parents[1] / 'surfaceproperties' / Path(file).with_suffix('.vsurf').name for surfaceproperty in self.all_surfaces.get(file, ()): @@ -545,7 +545,7 @@ def after_all_converted(self): continue vsurf['SurfacePropertiesList'].append(surfaceproperty) - vsurf_path.write_text(vsurf.ToString()) + keyvalues3.write(vsurf, vsurf_path) print("+ Saved", vsurf_path.local) if __name__ == '__main__': From 13a8e3d7870ddbf9f740d4b61855dc09cd45fff3 Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:40:11 +0200 Subject: [PATCH 5/9] fix --- utils/scripts_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts_import.py b/utils/scripts_import.py index de8c4c4..5955261 100644 --- a/utils/scripts_import.py +++ b/utils/scripts_import.py @@ -185,7 +185,7 @@ def ImportSoundscapesToVdata(file: Path): } } """ - keyvalues3.write(dict(data=(data=sndscape_data)), sndscape_file) + keyvalues3.write(dict(data=sndscape_data)), sndscape_file) print("+ Saved", sndscape_file.local) return sndscape_folder From ab28139306f9dec657476dc09266c6141416415f Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:41:47 +0200 Subject: [PATCH 6/9] fix v2 --- utils/scripts_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts_import.py b/utils/scripts_import.py index 5955261..b42e710 100644 --- a/utils/scripts_import.py +++ b/utils/scripts_import.py @@ -185,7 +185,7 @@ def ImportSoundscapesToVdata(file: Path): } } """ - keyvalues3.write(dict(data=sndscape_data)), sndscape_file) + keyvalues3.write(dict(data=sndscape_data), sndscape_file) print("+ Saved", sndscape_file.local) return sndscape_folder From ac935f5e53ed458aee666b35616a04c41c007265 Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:44:31 +0200 Subject: [PATCH 7/9] fix v4 --- utils/models_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/models_import.py b/utils/models_import.py index 1f9e2a3..c491cf8 100644 --- a/utils/models_import.py +++ b/utils/models_import.py @@ -2,7 +2,7 @@ import shutil from typing import Literal, Type, Union import shared.base_utils2 as sh -import kv3 +import keyvalues3 as kv3 from pathlib import Path from itertools import tee from srctools import smd From 1388102090c5b7ed5afcad04f2c1bf2538801f40 Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:47:26 +0200 Subject: [PATCH 8/9] fix 5 --- utils/particles_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/particles_import.py b/utils/particles_import.py index f3176eb..02117e7 100644 --- a/utils/particles_import.py +++ b/utils/particles_import.py @@ -1,5 +1,6 @@ import shared.base_utils2 as sh import shared.datamodel as dmx +from uuid import UUID import keyvalues3 as kv3 from dataclasses import dataclass from pathlib import Path From 0e6531a1aa291453e67284eb1d7d7be7f62e8dc8 Mon Sep 17 00:00:00 2001 From: Kristi K Date: Wed, 26 Apr 2023 20:58:17 +0200 Subject: [PATCH 9/9] bump to the one that accepts write --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3bd23a9..4333d7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ Pillow numpy # Particles, prop_data -keyvalues3==0.1a1 +keyvalues3==0.1a2 # Maps vdf