From 34f7e0313209735544066792037b04f8ee6c2736 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 20:42:56 -0400 Subject: [PATCH 1/7] Raise didactic/panproto floors and add lairs dependency Bump the minimum didactic to >=0.9.0 and panproto to >=0.56.0, and add lairs>=0.5.0 (resolved from the local sibling via tool.uv.sources until it is published). Importing bead does not import lairs. --- pyproject.toml | 11 ++- uv.lock | 250 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 245 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 70becde..2826c55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,9 @@ classifiers = [ ] requires-python = ">=3.14" dependencies = [ - "didactic>=0.7.2", - "panproto>=0.51.0", + "didactic>=0.9.0", + "panproto>=0.56.0", + "lairs>=0.5.0", "pyyaml>=6.0.0", "jinja2>=3.0.0", "uuid-utils>=0.7.0", @@ -188,3 +189,9 @@ exclude = [ dev = [ "hypothesis>=6.155.0", ] + +# lairs 0.5.0 (built on didactic 0.9 / panproto 0.56) is not yet on PyPI; resolve +# it from the local sibling checkout during development. uv ignores this source +# when bead is published, leaving the plain ``lairs>=0.5.0`` requirement. +[tool.uv.sources] +lairs = { path = "../lairs", editable = true } diff --git a/uv.lock b/uv.lock index d17eae4..fd25d19 100644 --- a/uv.lock +++ b/uv.lock @@ -155,6 +155,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] +[[package]] +name = "bases" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, + { name = "typing-validation" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/8d/105bca352e2fc5f1ee07f425ec296aa680525aac7f197ef135ea057902ac/bases-0.3.0.tar.gz", hash = "sha256:70f04a4a45d63245787f9e89095ca11042685b6b64b542ad916575ba3ccd1570", size = 789978, upload-time = "2023-12-18T16:57:17.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/15/7bcf28a3f971e1b0523fab46ae3ca935a589249544187558e5a8e70af393/bases-0.3.0-py3-none-any.whl", hash = "sha256:a2fef3366f3e522ff473d2e95c21523fe8e44251038d5c6150c01481585ebf5b", size = 36053, upload-time = "2023-12-18T16:57:14.253Z" }, +] + [[package]] name = "bead" version = "0.6.0" @@ -168,6 +181,7 @@ dependencies = [ { name = "glazing" }, { name = "jinja2" }, { name = "krippendorff" }, + { name = "lairs" }, { name = "langcodes" }, { name = "lark" }, { name = "pandas" }, @@ -242,19 +256,20 @@ requires-dist = [ { name = "anthropic", marker = "extra == 'api'", specifier = ">=0.8.0" }, { name = "click", specifier = ">=8.0.0" }, { name = "datasets", specifier = ">=2.14.0" }, - { name = "didactic", specifier = ">=0.7.2" }, + { name = "didactic", specifier = ">=0.9.0" }, { name = "evaluate", specifier = ">=0.4.0" }, { name = "glazing", specifier = ">=0.2.0" }, { name = "google-generativeai", marker = "extra == 'api'", specifier = ">=0.3.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.155.0" }, { name = "jinja2", specifier = ">=3.0.0" }, { name = "krippendorff", specifier = ">=0.6.0" }, + { name = "lairs", editable = "../lairs" }, { name = "langcodes", specifier = ">=3.3.0" }, { name = "lark", specifier = ">=1.0.0" }, { name = "openai", marker = "extra == 'api'", specifier = ">=1.0.0" }, { name = "pandas", specifier = ">=2.0.0" }, { name = "pandas-stubs", marker = "extra == 'dev'", specifier = ">=2.0.0" }, - { name = "panproto", specifier = ">=0.51.0" }, + { name = "panproto", specifier = ">=0.56.0" }, { name = "peft", specifier = ">=0.6.0" }, { name = "polars", specifier = ">=0.19.0" }, { name = "prompt-toolkit", specifier = ">=3.0.0" }, @@ -527,15 +542,15 @@ wheels = [ [[package]] name = "didactic" -version = "0.7.2" +version = "0.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, { name = "panproto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/e1/3694b7de53f9a09ee4a76f8496523362ffc25f913b0a958e4975452f22a5/didactic-0.7.2.tar.gz", hash = "sha256:279e4495908635f7facb41295fcb8122c7655cb85e758ece5453870132d5975b", size = 111030, upload-time = "2026-05-19T16:28:24.559Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/99/81e7f58dab442356340037ee7b32d61208f45f25f46faf7b8e83aeba758a/didactic-0.9.0.tar.gz", hash = "sha256:6cc669e88c65bf4b3f30ac37d07c4463dc055e04e16bd784f3ef2bc13b43bb8f", size = 121784, upload-time = "2026-06-17T17:31:48.592Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/c5/2db5aa15b3f83f4590273fae274f9f80e98e185eae7e9054a0a08a6b6ca8/didactic-0.7.2-py3-none-any.whl", hash = "sha256:8c314c7308d7cb15efe7382569c80d3999565a2ce26fbe6c24be070e069754c3", size = 136610, upload-time = "2026-05-19T16:28:22.881Z" }, + { url = "https://files.pythonhosted.org/packages/03/85/22b3e40af79f04e23a592bfb76ac42bb7dbae314f055f4fc7f2b30c8c86a/didactic-0.9.0-py3-none-any.whl", hash = "sha256:4de3c57e149211c34fb621691b3e61663449cafbcb1af5f4bb932ae34df9d7eb", size = 149494, upload-time = "2026-06-17T17:31:47.168Z" }, ] [[package]] @@ -565,6 +580,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, ] +[[package]] +name = "duckdb" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/29/9bad86ed7aa812d8c822a27c15c355b6d5423b991feeec86ed18027b6daa/duckdb-1.5.4.tar.gz", hash = "sha256:f9e32f1cdd106793d79d190186bed9e75289d51e68bd9174e47c04bffedeab6f", size = 18046634, upload-time = "2026-06-17T10:48:52.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/01/67ac4cbc8e552a1e14c029b5c443d828e68f94d5d913c574f577e1db277e/duckdb-1.5.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:4647968629d0677bbcc2416c7aeda8685eb84e4ca15a6dbd4f82a66cfc91a532", size = 32714364, upload-time = "2026-06-17T10:48:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/eb44d983fa56b175f971eea251bde284a36d26cbb93fcb68035061f54078/duckdb-1.5.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e8fcef301cf68d3951ea1eb8ac4d76cea0a6f6a08f4c78fe4026fc96d217bebc", size = 17349820, upload-time = "2026-06-17T10:48:37.126Z" }, + { url = "https://files.pythonhosted.org/packages/10/b2/b9dc7624b105d414585b8530451c1162c0b4750c0be9be2e497bb47a8a9b/duckdb-1.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f6f39cd0dc6948dee17fd130aec55114f97a8ef6e1db519b9774087962bc5c8c", size = 15498160, upload-time = "2026-06-17T10:48:40.032Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/61356444f6a8c62dec3c3d129abfc53f428de1d484093d1bb381db441231/duckdb-1.5.4-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:262f068158beb5943f2c618f4e54b46db8306b959f90dce956f90a89f613673d", size = 19374183, upload-time = "2026-06-17T10:48:42.698Z" }, + { url = "https://files.pythonhosted.org/packages/b0/f4/d5d633dd7c5138d8f7c434e6ac2553c831b7fb658494efa8d0bc73df8623/duckdb-1.5.4-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d2307a76d199077b0055b354e90e857479461a0d875437535dd4833172c8b6d", size = 21487202, upload-time = "2026-06-17T10:48:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/c0/26/5be13bbd5c3421dccfc1ad4ca9da4b97c5a3ddd73f66542092f3167ec52c/duckdb-1.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:6dcbb81a1276bc48deb4d562bce4f8895e4fc6348750a096e30052345c6d6552", size = 13666989, upload-time = "2026-06-17T10:48:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/dc/82/4d52f3f9f9703a226b26b80bdae3f6905aeefe5221bf1815fc93ff02ca25/duckdb-1.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:0f8722346024e5d9f02b58bf7b0491a629f97fdc8a04a10e432940f471ee387a", size = 14449863, upload-time = "2026-06-17T10:48:50.18Z" }, +] + [[package]] name = "emoji" version = "2.15.0" @@ -1010,6 +1040,114 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/4f/597b65bc88a031ee68b459fec05f8b40a60c3c3f6d5cde5b1403d965d199/krippendorff-0.8.2-py3-none-any.whl", hash = "sha256:1922ff73f7caa5faab6237a1abcb4f3e6ab605e3f7d39acd7998176873b8cd57", size = 18616, upload-time = "2025-11-03T03:03:38.583Z" }, ] +[[package]] +name = "lairs" +version = "0.5.0" +source = { editable = "../lairs" } +dependencies = [ + { name = "didactic" }, + { name = "duckdb" }, + { name = "httpx" }, + { name = "libipld" }, + { name = "multiformats" }, + { name = "panproto" }, + { name = "pyarrow" }, + { name = "textual" }, + { name = "websockets" }, +] + +[package.metadata] +requires-dist = [ + { name = "av", marker = "extra == 'video'" }, + { name = "brat", marker = "extra == 'brat'" }, + { name = "conllu", marker = "extra == 'conllu'" }, + { name = "datasets", marker = "extra == 'hf'" }, + { name = "decord", marker = "python_full_version < '3.14' and extra == 'video'" }, + { name = "didactic", specifier = ">=0.9.0" }, + { name = "duckdb" }, + { name = "fsspec", marker = "extra == 'fsspec'" }, + { name = "geopandas", marker = "extra == 'geo'" }, + { name = "glazing", marker = "extra == 'lexical'", specifier = ">=0.2.3" }, + { name = "habanero", marker = "extra == 'scholarly'" }, + { name = "httpx" }, + { name = "httpx", marker = "extra == 'appview'" }, + { name = "httpx", marker = "extra == 'reconciliation'" }, + { name = "huggingface-hub", marker = "extra == 'hf'" }, + { name = "label-studio-sdk", marker = "extra == 'labelstudio'" }, + { name = "lairs", extras = ["fsspec"], marker = "extra == 'ml'" }, + { name = "lairs", extras = ["hf"], marker = "extra == 'ml'" }, + { name = "lairs", extras = ["torch"], marker = "extra == 'ml'" }, + { name = "libipld" }, + { name = "librosa", marker = "extra == 'audio'" }, + { name = "mlflow", marker = "extra == 'tracking'" }, + { name = "mne", marker = "extra == 'neural'" }, + { name = "multiformats" }, + { name = "obonet", marker = "extra == 'bio'" }, + { name = "panproto", specifier = ">=0.56.0" }, + { name = "penman", marker = "extra == 'amr'" }, + { name = "polars", marker = "extra == 'polars'" }, + { name = "pyalex", marker = "extra == 'scholarly'" }, + { name = "pyarrow" }, + { name = "pympi-ling", marker = "extra == 'elan'" }, + { name = "qwikidata", marker = "extra == 'wikidata'" }, + { name = "shapely", marker = "extra == 'geo'" }, + { name = "soundfile", marker = "extra == 'audio'" }, + { name = "spacy", marker = "extra == 'spacy'" }, + { name = "sparqlwrapper", marker = "extra == 'wikidata'" }, + { name = "tensorflow", marker = "python_full_version < '3.14' and extra == 'tf'" }, + { name = "textgrid", marker = "extra == 'textgrid'" }, + { name = "textual", specifier = ">=8" }, + { name = "torch", marker = "extra == 'torch'" }, + { name = "wandb", marker = "extra == 'tracking'" }, + { name = "webdataset", marker = "extra == 'webdataset'" }, + { name = "websockets", specifier = ">=13" }, + { name = "xarray", marker = "extra == 'neural'" }, +] +provides-extras = ["hf", "torch", "tf", "webdataset", "polars", "fsspec", "conllu", "brat", "elan", "textgrid", "amr", "spacy", "labelstudio", "wikidata", "lexical", "reconciliation", "scholarly", "geo", "bio", "audio", "video", "neural", "tracking", "appview", "ml"] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis" }, + { name = "lairs", extras = ["amr"] }, + { name = "lairs", extras = ["appview"] }, + { name = "lairs", extras = ["audio"] }, + { name = "lairs", extras = ["bio"] }, + { name = "lairs", extras = ["brat"] }, + { name = "lairs", extras = ["conllu"] }, + { name = "lairs", extras = ["elan"] }, + { name = "lairs", extras = ["fsspec"] }, + { name = "lairs", extras = ["geo"] }, + { name = "lairs", extras = ["hf"] }, + { name = "lairs", extras = ["lexical"] }, + { name = "lairs", extras = ["neural"] }, + { name = "lairs", extras = ["polars"] }, + { name = "lairs", extras = ["reconciliation"] }, + { name = "lairs", extras = ["scholarly"] }, + { name = "lairs", extras = ["spacy"] }, + { name = "lairs", extras = ["textgrid"] }, + { name = "lairs", extras = ["tf"] }, + { name = "lairs", extras = ["torch"] }, + { name = "lairs", extras = ["tracking"] }, + { name = "lairs", extras = ["video"] }, + { name = "lairs", extras = ["webdataset"] }, + { name = "lairs", extras = ["wikidata"] }, + { name = "numpy", specifier = ">=2.3" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pytest-recording" }, + { name = "ruff" }, + { name = "ty" }, +] +docs = [ + { name = "griffe", specifier = ">=1.5" }, + { name = "mkdocs", specifier = ">=1.6" }, + { name = "mkdocs-autorefs", specifier = ">=1.2" }, + { name = "mkdocs-cinder", specifier = ">=1.2" }, + { name = "mkdocstrings", extras = ["python"], specifier = ">=0.27" }, + { name = "pygments", specifier = ">=2.18" }, + { name = "pymdown-extensions", specifier = ">=10.7" }, +] + [[package]] name = "langcodes" version = "3.5.1" @@ -1061,6 +1199,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/5e/3fb67e882c1fee01ebb7abc1c0a6669e5ff8acd060e93bfe7229e9ce6e4f/levenshtein-0.27.3-cp314-cp314t-win_arm64.whl", hash = "sha256:1d8520b89b7a27bb5aadbcc156715619bcbf556a8ac46ad932470945dca6e1bd", size = 91020, upload-time = "2025-11-01T12:14:22.944Z" }, ] +[[package]] +name = "libipld" +version = "3.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/f8/4ca4fe0d427481795f259045e6eb6e18d5a2d22efc91011feae5b9b6f18f/libipld-3.4.1.tar.gz", hash = "sha256:5d34fc8f90c954320fca7fef120e5fe9af58fb674ff7235f27870364de6c106f", size = 4578999, upload-time = "2026-05-27T17:42:37.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/c3/401f0f0c0e1c377b6dda3cabbdc1a207188cf1db4569ab67e5c0ed98a13e/libipld-3.4.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:7bd0d5920e7ff11ede281a90a729f987209f58c42de18661a159737445c66fa3", size = 261996, upload-time = "2026-05-27T17:41:30.016Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b8/77424fe62bbec5ca5bcac6340ac8a78c3df0b34c5a6c4c79739b732ccefb/libipld-3.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f736ae265e6e5f59542fd9bc20e9df610b21a0ebf4cfe567f296a93d4196a37a", size = 259630, upload-time = "2026-05-27T17:41:31.523Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c3/313ba9d12c6258339ef4572ba63b7c50bbeb1aa857736d0fd380e76d3763/libipld-3.4.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e30bb560954e34776367936d3f705c4339ead2e1549ad07e5b95fc389f8c368", size = 274792, upload-time = "2026-05-27T17:41:32.914Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fd/18ccfadc246e0957587206b13e9ee52a6816605cedf2a71867f2c9eec842/libipld-3.4.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2ba0268a158724ccb30f40b95c81edead520b9d23e38178bd671d8fe61cdf73a", size = 282308, upload-time = "2026-05-27T17:41:34.424Z" }, + { url = "https://files.pythonhosted.org/packages/23/ae/aea46f35b350bbd8c4ea28ca9af4645186a51346b75993d6c9e6ee80ce6f/libipld-3.4.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be013921f319efbe09a7d126d419723a48176ef1ce861d7a58cf2aa73871c69b", size = 307983, upload-time = "2026-05-27T17:41:35.793Z" }, + { url = "https://files.pythonhosted.org/packages/ce/16/a55ba44182d713b7a499fbf1f02c544384441b8b119737e2238178b38ddd/libipld-3.4.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c817f590edf5dbea8d2392645ed9e05f11e696ba9cd5ed7199b3f78740e73767", size = 319535, upload-time = "2026-05-27T17:41:37.631Z" }, + { url = "https://files.pythonhosted.org/packages/a8/02/04ce510e24cbc2bd1830787492e455a47c3f0a42a3d25ccccfb50b7d3eb4/libipld-3.4.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86b887423cf756c8711184d73311c9254b6a9a8007af54d6a7aa9f1fac978353", size = 284427, upload-time = "2026-05-27T17:41:39.106Z" }, + { url = "https://files.pythonhosted.org/packages/56/3f/329225b873b15eb47c9d1501462993a3b87272f74945e2e423ca2dfee8f1/libipld-3.4.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:663be567595acb999a61b32ad0d8755126178e9506f300939f408e17d1bcf642", size = 300937, upload-time = "2026-05-27T17:41:40.491Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ca/89b5d65aaab48539f5a04c0b2edb4f29f57c536e7219a4704003389b346c/libipld-3.4.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:03b2a76abb21960f5d2af13400a76246376e3c5bc8a2517e8e8fceb7f7e56441", size = 451215, upload-time = "2026-05-27T17:41:41.917Z" }, + { url = "https://files.pythonhosted.org/packages/38/50/36ee9838774a8576120c59fb1860b222aad587156b29b36df242e4b58b08/libipld-3.4.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:a223a8fd7a6a37259e45169d5a1bd7a0c4ee43c7377a7f8a0a77550247b531d9", size = 489969, upload-time = "2026-05-27T17:41:43.637Z" }, + { url = "https://files.pythonhosted.org/packages/74/69/1ee85b8a024e2fd2049fb20e1158b61cdd97452b1bec54169796ba99e994/libipld-3.4.1-cp314-cp314-win32.whl", hash = "sha256:6eab7b7a9279f66837c2028a072b8150635a69a28d6f9fda85b38aa8ae275b01", size = 156242, upload-time = "2026-05-27T17:41:45.051Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/091032af546c828bcd25859404534c8ac93897aa884800841a5358b3cb02/libipld-3.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:294d076f08e13039e21a5fe8a791d60ef2798c5000ab146cbe86a06f38acb466", size = 166100, upload-time = "2026-05-27T17:41:46.649Z" }, + { url = "https://files.pythonhosted.org/packages/53/be/6e949dfa7245bcddce25a7ef2c73590e104403f7816f2ee3a0fe37fa2e09/libipld-3.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:4f49e81ba70b43d0d55e94e71d2fdb34b97d3c62af2859c98dd385a6b884bceb", size = 150208, upload-time = "2026-05-27T17:41:48.017Z" }, +] + [[package]] name = "lightning-utilities" version = "0.15.2" @@ -1262,6 +1421,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" }, ] +[[package]] +name = "multiformats" +version = "0.3.1.post4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bases" }, + { name = "multiformats-config" }, + { name = "typing-extensions" }, + { name = "typing-validation" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/41/2efc6e99fa2ed9f1a47fbfed5d124215e35db0a849585db72eeb1490de0e/multiformats-0.3.1.post4.tar.gz", hash = "sha256:d00074fdbc7d603c2084b4c38fa17bbc28173cf2750f51f46fbbc5c4d5605fbb", size = 826017, upload-time = "2023-12-20T14:18:00.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/02/0eff41b136c6723441d052c61c9dae36b86b3ae68ec064813445580222a6/multiformats-0.3.1.post4-py3-none-any.whl", hash = "sha256:5b1d61bd8275c9e817bdbee38dbd501b26629011962ee3c86c46e7ccd0b14129", size = 57148, upload-time = "2023-12-20T14:17:58.576Z" }, +] + +[[package]] +name = "multiformats-config" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "multiformats" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/09/ccb6867c2d6c6de98d1d285d8a3a2103fdf452c2fef5019bb3d8ac9938d9/multiformats-config-0.3.1.tar.gz", hash = "sha256:7eaa80ef5d9c5ee9b86612d21f93a087c4a655cbcb68960457e61adbc62b47a7", size = 28345, upload-time = "2023-12-18T21:35:23.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/9b/c21a9c1d5ea4847989f1eb00e3147e38e79aaea7c4b4d1cbd4f1afae9740/multiformats_config-0.3.1-py3-none-any.whl", hash = "sha256:dec4c9d42ed0d9305889b67440f72e8e8d74b82b80abd7219667764b5b0a8e1d", size = 17153, upload-time = "2023-12-18T21:35:21.171Z" }, +] + [[package]] name = "multiprocess" version = "0.70.18" @@ -1565,14 +1751,14 @@ wheels = [ [[package]] name = "panproto" -version = "0.51.0" +version = "0.56.1" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/dd/9810ef2efa8335dd527db653214d791a852b896d2a0a6a8c8e9f20ed656c/panproto-0.51.0-cp313-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d97a690159814cb482eb085494cb7b881a30e047c0d2847d88bf97203a3104bd", size = 11726180, upload-time = "2026-05-28T21:24:21.595Z" }, - { url = "https://files.pythonhosted.org/packages/12/99/8979129533b05012be4e6cbdc16b94ba3ed861b371c54606fd2e9c156b38/panproto-0.51.0-cp313-abi3-macosx_11_0_arm64.whl", hash = "sha256:69a68c137dc0c9d5bb7525fec907b4283316d512b0c77ca910ccd7eb6ca7f643", size = 11471081, upload-time = "2026-05-28T21:24:24.356Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5f/deba52da6c9e15688f4d99651977d3f581d10486100f13a517763c7e6e38/panproto-0.51.0-cp313-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6d735f7d1f1b1685dd9fd7466bf0bdd93d034e511a5d702228664c2966c9415a", size = 12016308, upload-time = "2026-05-28T21:24:27.323Z" }, - { url = "https://files.pythonhosted.org/packages/49/08/0893f0852a0c71eb59c7f76ed9cf321dad668377fee8714f88c3c631ad51/panproto-0.51.0-cp313-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b22ec1545ea0ba615683864b159bc3832eb311d0241d9fef18414b919980de50", size = 12599142, upload-time = "2026-05-28T21:24:30.547Z" }, - { url = "https://files.pythonhosted.org/packages/69/7d/966f47265f5048a86276938769a51946c05e745c40ce96d81d615b2f479f/panproto-0.51.0-cp313-abi3-win_amd64.whl", hash = "sha256:08c55da3a1659718b6b85be65c5b438567721b84e40105d03862b10b743d6235", size = 11678138, upload-time = "2026-05-28T21:24:33.38Z" }, + { url = "https://files.pythonhosted.org/packages/0a/08/b73062928417d9cc373f7d8eb43050a4f3486c083651c217204ede2b1c2a/panproto-0.56.1-cp313-abi3-macosx_10_12_x86_64.whl", hash = "sha256:481fba15f19fccb400595957d90c518e0ea95bfbae6f8702b64c063bd07e4ec4", size = 10511858, upload-time = "2026-06-18T17:47:55.144Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e1/2e4936b80ef33a6d5e9130670e5ba874f711c8b79b7f4b76a74849637c23/panproto-0.56.1-cp313-abi3-macosx_11_0_arm64.whl", hash = "sha256:44041e3844703baa8db1056906e7e847104335a7fa541d9e9e49b91d054aea09", size = 9901815, upload-time = "2026-06-18T17:47:58.045Z" }, + { url = "https://files.pythonhosted.org/packages/ce/da/c5a849418ebf713ce34b32ee9982f1c61eb47b921be0e738a868d7743653/panproto-0.56.1-cp313-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:726d094e58e695019b16ae9b75ad2a9380f25a04a7ece187b3c27425ca3e4131", size = 9955380, upload-time = "2026-06-18T17:48:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/ed/4a/8ccf4abd96414d54628932ea9f909dd9bfa1ee0c13b4b0467a7f85046b2a/panproto-0.56.1-cp313-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cd6efefcd216c9e6e004872ea220ac4ae9d6a80b7bea851def04a1319b8eb4ef", size = 10694495, upload-time = "2026-06-18T17:48:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/be/df/2dbb9ea60c6216a8e617bd48b32c357c6314bbfa7726ee2c0ec367e121ec/panproto-0.56.1-cp313-abi3-win_amd64.whl", hash = "sha256:2e91753465b2b77f17a27886ce49629d375ead80a9aa6ec67dc5ec29f78b303a", size = 11760683, upload-time = "2026-06-18T17:48:04.486Z" }, ] [[package]] @@ -2590,7 +2776,7 @@ wheels = [ [[package]] name = "textual" -version = "7.3.0" +version = "8.2.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py", extra = ["linkify"] }, @@ -2600,9 +2786,9 @@ dependencies = [ { name = "rich" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6f/ee/620c887bfad9d6eba062dfa3b6b0e735e0259102e2667b19f21625ef598d/textual-7.3.0.tar.gz", hash = "sha256:3169e8ba5518a979b0771e60be380ab1a6c344f30a2126e360e6f38d009a3de4", size = 1590692, upload-time = "2026-01-15T16:32:02.342Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/7a/c519db0aba5024f86e71e9631810bfdd6866ed2c8695bd7fa34b90e7ef59/textual-8.2.7.tar.gz", hash = "sha256:658f568ff81e30ed43890c3e07520390e5cf1b4763822006e060656b0a88f105", size = 1859249, upload-time = "2026-05-19T10:52:49.531Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/1f/abeb4e5cb36b99dd37db72beb2a74d58598ccb35aaadf14624ee967d4a6b/textual-7.3.0-py3-none-any.whl", hash = "sha256:db235cecf969c87fe5a9c04d83595f506affc9db81f3a53ab849534d726d330a", size = 716374, upload-time = "2026-01-15T16:31:58.233Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f5/c1e18bc0707300a0e90204343abbf7d7acd6fb7ebe03a6d4893b99a234b8/textual-8.2.7-py3-none-any.whl", hash = "sha256:4caaa13a90bc4cf9c6c862c067ccd34fe84e9c161710a2a907a8026313b6bd73", size = 731129, upload-time = "2026-05-19T10:52:51.773Z" }, ] [[package]] @@ -2828,6 +3014,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "typing-validation" +version = "1.2.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/c0/374639373a99b62f51c3204521235906dca1ed1886e73f65d6664465b187/typing_validation-1.2.12.tar.gz", hash = "sha256:7ea9463a18bd04922e799cac1954f687e68e9564773f81db491536852ffe1d54", size = 774523, upload-time = "2025-03-18T14:54:49.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/7b/29a088c5be56f40e0b1e611c460681f411ce79f0083d2cd3b233a35b7c4d/typing_validation-1.2.12-py3-none-any.whl", hash = "sha256:d68e22a41bf2b98ae91e5d6407db56e9ef83e9e5600164a7aff64aaa082fc232", size = 20657, upload-time = "2025-03-18T14:54:47.529Z" }, +] + [[package]] name = "tzdata" version = "2025.3" @@ -2939,6 +3134,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/74/a148b41572656904a39dfcfed3f84dd1066014eed94e209223ae8e9d088d/weasel-0.4.3-py3-none-any.whl", hash = "sha256:08f65b5d0dbded4879e08a64882de9b9514753d9eaa4c4e2a576e33666ac12cf", size = 50757, upload-time = "2025-11-13T23:52:26.982Z" }, ] +[[package]] +name = "websockets" +version = "16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, + { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, + { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, + { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, + { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, + { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, +] + [[package]] name = "werkzeug" version = "3.1.5" From 7fa4a2910ae9dba6ee883317d38b7fe434c4f3f0 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 20:43:56 -0400 Subject: [PATCH 2/7] Consolidate layers interop onto canonical lairs models Delete the hand-mirrored layers models and the snake<->camel / MirrorIso machinery, and re-point the corpus-record, graph, parse, and resource lenses onto the generated lairs.records models: each lens now returns a typed lairs model (or a small bead-side aggregate of them) as its view, with the bead-only remainder in a JsonValue complement, so the GetPut/PutGet laws still hold. Retire the redundant Node lexicon validator and the vendor/layers submodule; validation is now inherent in the lairs generated models. Migrate the interop tests accordingly. --- .github/workflows/ci.yml | 2 - .gitmodules | 5 - bead/interop/layers/__init__.py | 52 ++-- bead/interop/layers/_convert.py | 170 ++++++----- bead/interop/layers/_mirror.py | 115 ------- bead/interop/layers/bridges.py | 58 ++-- bead/interop/layers/graph_lens.py | 192 ++++++------ bead/interop/layers/model_lenses.py | 154 ---------- bead/interop/layers/models.py | 289 ------------------ bead/interop/layers/models_records.py | 252 --------------- bead/interop/layers/parse_lens.py | 203 ++++++------ bead/interop/layers/resource_lens.py | 216 ++++++------- tests/interop/lexicon_validator/.gitignore | 2 - tests/interop/lexicon_validator/package.json | 9 - tests/interop/lexicon_validator/validate.mjs | 45 --- tests/interop/test_layers_coverage.py | 73 ----- tests/interop/test_layers_defs.py | 140 --------- tests/interop/test_layers_graph_roundtrip.py | 14 +- .../interop/test_layers_lexicon_validation.py | 229 -------------- tests/interop/test_layers_parse_iso.py | 41 +-- tests/interop/test_layers_record_bridge.py | 8 +- tests/interop/test_layers_records.py | 134 -------- tests/interop/test_layers_resource.py | 20 +- vendor/layers | 1 - 24 files changed, 523 insertions(+), 1901 deletions(-) delete mode 100644 .gitmodules delete mode 100644 bead/interop/layers/_mirror.py delete mode 100644 bead/interop/layers/model_lenses.py delete mode 100644 bead/interop/layers/models.py delete mode 100644 bead/interop/layers/models_records.py delete mode 100644 tests/interop/lexicon_validator/.gitignore delete mode 100644 tests/interop/lexicon_validator/package.json delete mode 100644 tests/interop/lexicon_validator/validate.mjs delete mode 100644 tests/interop/test_layers_coverage.py delete mode 100644 tests/interop/test_layers_defs.py delete mode 100644 tests/interop/test_layers_lexicon_validation.py delete mode 100644 tests/interop/test_layers_records.py delete mode 160000 vendor/layers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 071bd3e..f18f968 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,8 +180,6 @@ jobs: needs: [ts-build] # Python tests need TypeScript compiled steps: - uses: actions/checkout@v4 - with: - submodules: true # vendor/layers lexicons for interop validation - name: Install pnpm uses: pnpm/action-setup@v4 diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a1bc115..0000000 --- a/.gitmodules +++ /dev/null @@ -1,5 +0,0 @@ -[submodule "vendor/layers"] - path = vendor/layers - url = https://github.com/layers-pub/layers.git - branch = main - shallow = true diff --git a/bead/interop/layers/__init__.py b/bead/interop/layers/__init__.py index 56f7b36..69571a4 100644 --- a/bead/interop/layers/__init__.py +++ b/bead/interop/layers/__init__.py @@ -1,20 +1,22 @@ """Lossless, law-verified lenses between bead models and the ``layers`` schema. -Maps bead's corpus and annotation models to ``layers``-shaped JSON and back via -didactic lenses (``dx.Lens``/``dx.Iso``): the layers view is a faithful, -standalone projection; the lens complement holds the bead-only round-trip -remainder. Round-trip fidelity is guaranteed by the didactic GetPut/PutGet laws. +Maps bead's corpus, annotation, and resource models to and from the canonical +``layers`` record models generated by ``lairs`` (``lairs.records``) via didactic +lenses (``dx.Lens``/``dx.Iso``): the layers view is a faithful, standalone +projection built from the generated models; the lens complement holds the +bead-only round-trip remainder. Round-trip fidelity is guaranteed by the +didactic GetPut/PutGet laws. What is mapped: -- The linguistic ``layers`` constructs (the shared object definitions and the - expression, segmentation, annotation, graph, media, and ontology records) are - mirrored as didactic models with a generic ``MirrorIso`` (see ``models``, - ``models_records``, and ``model_lenses``). -- bead's pipeline outputs bridge directly to layers: ``CorpusGraph`` to the - property graph, ``CorpusRecord`` to an ``expression``, and a dependency parse - to a ``tokenization`` with part-of-speech and dependency annotation layers. -- bead's resources map to their layers counterparts: ``LexicalItem`` to an +- ``CorpusRecord`` to a layers ``expression`` (see ``bridges``). +- ``CorpusGraph`` to the layers property graph (expressions, graph nodes, and a + ``graphEdgeSet``) bundled as a :class:`CorpusGraphLayers` view (see + ``graph_lens``). +- a dependency parse to a layers ``tokenization`` with part-of-speech and + dependency annotation layers bundled as a :class:`ParsedSentenceLayers` view + (see ``parse_lens``). +- bead's resources to their layers counterparts: ``LexicalItem`` to an ``entry``, ``Lexicon`` to a ``collection``, and ``Template`` to a ``template`` (see ``resource_lens``). """ @@ -28,20 +30,13 @@ ) from bead.interop.layers.graph_lens import ( CORPUS_GRAPH_LAYERS, + CorpusGraphLayers, CorpusGraphLayersLens, graph_to_layers, ) -from bead.interop.layers.model_lenses import ( - ALL_MIRROR_ISOS, - RECORD_ISOS, - RECORD_MODELS, - SHARED_DEF_ISOS, - SHARED_DEF_MODELS, - MirrorIso, - mirror_iso, -) from bead.interop.layers.parse_lens import ( PARSED_SENTENCE_LAYERS, + ParsedSentenceLayers, ParsedSentenceLayersIso, parse_to_layers, ) @@ -51,30 +46,27 @@ TEMPLATE_LAYERS, LexicalItemEntryLens, LexiconCollectionLens, + LexiconLayers, TemplateLayersLens, ) __all__ = [ - "ALL_MIRROR_ISOS", "CORPUS_GRAPH_LAYERS", "LEXICAL_ITEM_ENTRY", "LEXICON_COLLECTION", "PARSED_SENTENCE_LAYERS", "RECORD_EXPRESSION", "TEMPLATE_LAYERS", + "CorpusGraphLayers", + "CorpusGraphLayersLens", "LexicalItemEntryLens", "LexiconCollectionLens", - "TemplateLayersLens", - "RECORD_ISOS", - "RECORD_MODELS", - "SHARED_DEF_ISOS", - "SHARED_DEF_MODELS", - "CorpusGraphLayersLens", - "MirrorIso", + "LexiconLayers", + "ParsedSentenceLayers", "ParsedSentenceLayersIso", "RecordExpressionLens", + "TemplateLayersLens", "graph_to_layers", - "mirror_iso", "parse_to_layers", "record_to_expression", ] diff --git a/bead/interop/layers/_convert.py b/bead/interop/layers/_convert.py index 6d56395..26279da 100644 --- a/bead/interop/layers/_convert.py +++ b/bead/interop/layers/_convert.py @@ -1,10 +1,13 @@ -"""Shared, reversible conversions between bead values and layers JSON shapes. +"""Shared, reversible conversions between bead values and ``lairs`` models. These helpers centralize the mechanical, lossless conversions every layers lens -relies on: feature maps, object references, confidence scaling, and capture / -restore of a bead model's framework identity (the ``BeadBaseModel`` id and -timestamps, which ``layers`` represents through its own identity scheme and so -travel in a lens complement rather than the layers view). +relies on. Two sinks are served: + +- the layers *view*, built from the canonical ``lairs.records`` models, uses the + typed :func:`feature_map` / :func:`object_ref` builders. +- the lens *complement*, a plain ``JsonValue``, carries the bead-only remainder + (framework identity and metadata dicts ``layers`` has no slot for) via + :func:`identity_of` and :func:`dumps_meta`. """ from __future__ import annotations @@ -15,24 +18,44 @@ from typing import TYPE_CHECKING from uuid import UUID +from lairs.records import defs + from bead.corpus.records import ProvenanceValue from bead.data.base import BeadBaseModel, JsonValue if TYPE_CHECKING: from bead.items.item import MetadataValue +# layers scales confidence to an integer 0-1000 (to avoid floats on the wire). +CONFIDENCE_SCALE = 1000 + +# Canonical layers collection NSIDs (record level), used both as fragment record +# NSIDs and as the collection segment of minted corpus AT-URIs. +EXPRESSION_NSID = "pub.layers.expression.expression" +SEGMENTATION_NSID = "pub.layers.segmentation.segmentation" +ANNOTATION_LAYER_NSID = "pub.layers.annotation.annotationLayer" +CORPUS_NSID = "pub.layers.corpus.corpus" +MEMBERSHIP_NSID = "pub.layers.corpus.membership" + -def to_feature_map(features: Mapping[str, MetadataValue]) -> JsonValue: - """Encode a feature dict as a layers ``featureMap`` (values JSON-encoded). +# --- typed feature maps (the layers view) ----------------------------------- + + +def feature_map(features: Mapping[str, MetadataValue]) -> defs.FeatureMap | None: + """Build a layers ``featureMap`` model, or ``None`` for an empty mapping. Each value is serialized with ``json.dumps`` so arbitrary (including - non-string) values round-trip exactly via :func:`from_feature_map`. Entries - preserve the dict's insertion order so the round-trip is exact. + non-string) values round-trip exactly via :func:`read_feature_map`. Entries + preserve the mapping's iteration order so the round-trip is exact. An empty + mapping projects to ``None`` (a faithful layers view omits empty optionals). """ - entries: tuple[JsonValue, ...] = tuple( - {"key": key, "value": json.dumps(features[key])} for key in features + if not features: + return None + return defs.FeatureMap( + entries=tuple( + defs.Feature(key=key, value=json.dumps(features[key])) for key in features + ) ) - return {"entries": entries} type _Loaded = str | int | float | bool | None | list["_Loaded"] | dict[str, "_Loaded"] @@ -47,71 +70,61 @@ def _tuplify(value: _Loaded) -> MetadataValue: return value -def from_feature_map(feature_map: JsonValue) -> dict[str, MetadataValue]: - """Decode a layers ``featureMap`` back into a feature dict.""" - result: dict[str, MetadataValue] = {} - if not isinstance(feature_map, dict): - return result - entries = feature_map.get("entries") - if not isinstance(entries, tuple): - return result - for entry in entries: - if isinstance(entry, dict): - key = entry.get("key") - value = entry.get("value") - if isinstance(key, str) and isinstance(value, str): - result[key] = _tuplify(json.loads(value)) - return result - - -def from_feature_map_scalar(feature_map: JsonValue) -> dict[str, ProvenanceValue]: - """Decode a ``featureMap`` whose values are flat provenance scalars.""" - result: dict[str, ProvenanceValue] = {} - if not isinstance(feature_map, dict): - return result - entries = feature_map.get("entries") - if not isinstance(entries, tuple): - return result - for entry in entries: - if isinstance(entry, dict): - key = entry.get("key") - value = entry.get("value") - if isinstance(key, str) and isinstance(value, str): - result[key] = json.loads(value) - return result - - -def strip_nulls(value: JsonValue) -> JsonValue: - """Recursively drop dict entries whose value is ``None``. - - The ATProto data model has no null: optional fields are omitted, not set to - null, and a lexicon rejects an explicit null for a typed optional field. - Layers views therefore omit absent optionals; the round-trip is unaffected - because the reverse direction defaults missing keys back to ``None``. +def read_feature_map(fm: defs.FeatureMap | None) -> dict[str, MetadataValue]: + """Decode a layers ``featureMap`` model back into a feature dict.""" + if fm is None: + return {} + return {entry.key: _tuplify(json.loads(entry.value)) for entry in fm.entries} + + +def read_feature_map_scalar( + fm: defs.FeatureMap | None, +) -> dict[str, ProvenanceValue]: + """Decode a ``featureMap`` model whose values are flat provenance scalars.""" + if fm is None: + return {} + return {entry.key: json.loads(entry.value) for entry in fm.entries} + + +# --- typed object references (the layers view) ------------------------------ + + +def object_ref(local_id: str) -> defs.ObjectRef: + """Build a layers ``objectRef`` to a local object by id.""" + return defs.ObjectRef(localId=defs.Uuid(value=local_id)) + + +def from_object_ref(ref: defs.ObjectRef) -> str: + """Read the local id out of a layers ``objectRef`` model.""" + if ref.localId is None: + raise ValueError("objectRef has no localId") + return ref.localId.value + + +# --- metadata dicts in the lens complement (JsonValue) ---------------------- + + +def dumps_meta(mapping: Mapping[str, MetadataValue]) -> str: + """Encode a metadata dict as a JSON string for a lens complement. + + ``MetadataValue`` admits tuples (which are not ``JsonValue``); serializing to + a JSON string keeps the complement a plain ``JsonValue`` and round-trips + exactly through :func:`loads_meta`. Key insertion order is preserved so the + reconstructed dict compares equal to the original (didactic compares dict + fields order-sensitively). """ - if isinstance(value, dict): - return { - key: strip_nulls(item) for key, item in value.items() if item is not None - } - if isinstance(value, tuple): - return tuple(strip_nulls(item) for item in value) - return value + return json.dumps(dict(mapping)) -def object_ref(local_id: str) -> JsonValue: - """Build a layers ``objectRef`` to a local node by id.""" - return {"localId": {"value": local_id}} +def loads_meta(value: JsonValue) -> dict[str, MetadataValue]: + """Decode a :func:`dumps_meta` string back into a metadata dict.""" + loaded = json.loads(_as_str(value)) + if not isinstance(loaded, dict): + return {} + return {str(key): _tuplify(val) for key, val in loaded.items()} -def from_object_ref(ref: JsonValue) -> str: - """Read the local id out of a layers ``objectRef``.""" - if isinstance(ref, dict): - local = ref.get("localId") - if isinstance(local, dict): - value = local.get("value") - if isinstance(value, str): - return value - raise ValueError("objectRef has no localId.value") +# --- framework identity (the lens complement) ------------------------------- def identity_of(model: BeadBaseModel) -> JsonValue: @@ -143,6 +156,9 @@ def apply_identity[T: BeadBaseModel](model: T, identity: JsonValue) -> T: ) +# --- JsonValue narrowers (for reading complements) -------------------------- + + def _as_str(value: JsonValue) -> str: if not isinstance(value, str): raise ValueError(f"expected str, got {type(value).__name__}") @@ -157,9 +173,13 @@ def j_obj(value: JsonValue) -> dict[str, JsonValue]: def j_list(value: JsonValue) -> tuple[JsonValue, ...]: - """Narrow a ``JsonValue`` to a JSON array, raising otherwise.""" - if isinstance(value, tuple): - return value + """Narrow a ``JsonValue`` to a JSON array, raising otherwise. + + Accepts both tuples (in-process complements) and lists (complements that have + round-tripped through JSON, as in the codec), normalizing to a tuple. + """ + if isinstance(value, (tuple, list)): + return tuple(value) raise ValueError(f"expected JSON array, got {type(value).__name__}") diff --git a/bead/interop/layers/_mirror.py b/bead/interop/layers/_mirror.py deleted file mode 100644 index 7f349ef..0000000 --- a/bead/interop/layers/_mirror.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Generic serialization between mirror models and layers JSON. - -The mirror models in :mod:`bead.interop.layers.models` match the ``layers`` -schema structurally: snake_case fields correspond to layers' camelCase, nested -objects are embedded models, feature maps are :class:`FeatureMap`, and -confidence is an integer. A single pair of conversions therefore serializes any -of them to and from layers-shaped JSON. - -Conversion goes through each model's canonical JSON form (``model_dump_json`` / -``model_validate_json``), so it does not depend on didactic's internal -field-value types. -""" - -from __future__ import annotations - -import json -import re - -import didactic.api as dx - -from bead.data.base import JsonValue -from bead.interop.layers._convert import j_obj, strip_nulls - -_CAMEL_BOUNDARY = re.compile(r"([A-Z])") - -type _Loaded = str | int | float | bool | None | list["_Loaded"] | dict[str, "_Loaded"] - - -def _to_camel(name: str) -> str: - head, *rest = name.split("_") - return head + "".join(part[:1].upper() + part[1:] for part in rest) - - -def _to_snake(name: str) -> str: - return _CAMEL_BOUNDARY.sub(lambda match: "_" + match.group(1).lower(), name) - - -def _camel_keys(value: _Loaded) -> JsonValue: - """Recursively camelCase dict keys and turn JSON arrays into tuples.""" - if isinstance(value, dict): - return {_to_camel(key): _camel_keys(item) for key, item in value.items()} - if isinstance(value, list): - return tuple(_camel_keys(item) for item in value) - return value - - -def _snake_keys(value: JsonValue) -> JsonValue: - """Recursively snake_case dict keys (arrays stay tuples).""" - if isinstance(value, dict): - return {_to_snake(key): _snake_keys(item) for key, item in value.items()} - if isinstance(value, tuple): - return tuple(_snake_keys(item) for item in value) - return value - - -_DEFS_NSID = "pub.layers.defs" - -#: The ``externalTarget.selector`` union variants (camelCase mirror keys, each -#: also the layers def name). layers models this as an open ATProto union, so the -#: wire value carries a ``$type`` discriminator rather than a wrapper key. -_SELECTOR_VARIANTS = frozenset( - {"textQuoteSelector", "textPositionSelector", "fragmentSelector"} -) - - -def _wrap_unions(value: JsonValue) -> JsonValue: - """Rewrite ``selector`` wrappers into ATProto ``$type`` union members.""" - if isinstance(value, dict): - result: dict[str, JsonValue] = {} - for key, item in value.items(): - if key == "selector" and isinstance(item, dict) and len(item) == 1: - variant, payload = next(iter(item.items())) - if variant in _SELECTOR_VARIANTS and isinstance(payload, dict): - member: dict[str, JsonValue] = {"$type": f"{_DEFS_NSID}#{variant}"} - for inner_key, inner_item in payload.items(): - member[inner_key] = _wrap_unions(inner_item) - result[key] = member - continue - result[key] = _wrap_unions(item) - return result - if isinstance(value, tuple): - return tuple(_wrap_unions(item) for item in value) - return value - - -def _unwrap_unions(value: JsonValue) -> JsonValue: - """Rewrite ATProto ``$type`` selector union members back to wrappers.""" - if isinstance(value, dict): - result: dict[str, JsonValue] = {} - for key, item in value.items(): - type_ref = item.get("$type") if isinstance(item, dict) else None - if key == "selector" and isinstance(type_ref, str): - variant = type_ref.rsplit("#", 1)[-1] - payload: dict[str, JsonValue] = {} - for inner_key, inner_item in j_obj(item).items(): - if inner_key != "$type": - payload[inner_key] = _unwrap_unions(inner_item) - result[key] = {variant: payload} - continue - result[key] = _unwrap_unions(item) - return result - if isinstance(value, tuple): - return tuple(_unwrap_unions(item) for item in value) - return value - - -def mirror_to_layers(model: dx.Model) -> JsonValue: - """Serialize a faithful mirror model to layers-shaped JSON (camelCase).""" - return _wrap_unions(strip_nulls(_camel_keys(json.loads(model.model_dump_json())))) - - -def mirror_from_layers[M: dx.Model](model_type: type[M], data: JsonValue) -> M: - """Deserialize layers-shaped JSON back into a mirror model.""" - restored = _snake_keys(j_obj(_unwrap_unions(data))) - return model_type.model_validate_json(json.dumps(restored)) diff --git a/bead/interop/layers/bridges.py b/bead/interop/layers/bridges.py index fc0d30a..84c4bae 100644 --- a/bead/interop/layers/bridges.py +++ b/bead/interop/layers/bridges.py @@ -1,45 +1,46 @@ -"""Bridge lenses between bead-native models and layers constructs. +"""Bridge lens between a bead ``CorpusRecord`` and a layers ``expression``. -These map the things bead's pipeline actually produces onto layers records: - -- :class:`~bead.corpus.records.CorpusRecord` <-> a layers ``expression``. - -The layers view is a faithful, standalone projection; the lens complement holds -the bead-only remainder (framework identity and fields layers has no slot for), -so the round-trip is exact and the GetPut/PutGet laws hold. +A :class:`~bead.corpus.records.CorpusRecord` projects to a canonical +:class:`lairs.records.expression.Expression`; the lens complement holds the +bead-only remainder (framework identity, the source name, and the record index, +which ``layers`` has no slot for), so the round-trip is exact and the +GetPut/PutGet laws hold. """ from __future__ import annotations import didactic.api as dx +from lairs.records import expression from bead.corpus.records import CorpusRecord from bead.data.base import JsonValue from bead.interop.layers._convert import ( apply_identity, - from_feature_map_scalar, + feature_map, identity_of, j_int, j_obj, j_str, - to_feature_map, + read_feature_map_scalar, ) -_EXPRESSION_KIND = "expression" +# CorpusRecord has no kind field; corpus records are atomic, sentence-level text +# units, so the synthesized expression kind is a fixed, valid slug. +_EXPRESSION_KIND = "sentence" -class RecordExpressionLens(dx.Lens[CorpusRecord, JsonValue, JsonValue]): - """Lossless lens ``CorpusRecord <-> (layers expression view, complement)``.""" +class RecordExpressionLens(dx.Lens[CorpusRecord, expression.Expression, JsonValue]): + """Lossless lens ``CorpusRecord <-> (layers expression, complement)``.""" - def forward(self, record: CorpusRecord) -> tuple[JsonValue, JsonValue]: - """Project a record to a layers expression view and bead complement.""" - view: JsonValue = { - "id": str(record.id), - "kind": _EXPRESSION_KIND, - "text": record.text, - "features": to_feature_map(record.provenance), - "createdAt": record.created_at.isoformat(), - } + def forward(self, record: CorpusRecord) -> tuple[expression.Expression, JsonValue]: + """Project a record to a layers expression and bead complement.""" + view = expression.Expression( + id=str(record.id), + kind=_EXPRESSION_KIND, + createdAt=record.created_at, + text=record.text, + features=feature_map(record.provenance), + ) complement: JsonValue = { "identity": identity_of(record), "source_name": record.source_name, @@ -47,15 +48,16 @@ def forward(self, record: CorpusRecord) -> tuple[JsonValue, JsonValue]: } return view, complement - def backward(self, view: JsonValue, complement: JsonValue) -> CorpusRecord: - """Reconstruct a record from its layers expression view and complement.""" - view_obj = j_obj(view) + def backward( + self, view: expression.Expression, complement: JsonValue + ) -> CorpusRecord: + """Reconstruct a record from its layers expression and complement.""" comp = j_obj(complement) record = CorpusRecord( - text=j_str(view_obj["text"]), + text=view.text if view.text is not None else "", source_name=j_str(comp["source_name"]), record_index=j_int(comp["record_index"]), - provenance=from_feature_map_scalar(view_obj["features"]), + provenance=read_feature_map_scalar(view.features), ) return apply_identity(record, comp["identity"]) @@ -63,7 +65,7 @@ def backward(self, view: JsonValue, complement: JsonValue) -> CorpusRecord: RECORD_EXPRESSION = RecordExpressionLens() -def record_to_expression(record: CorpusRecord) -> JsonValue: +def record_to_expression(record: CorpusRecord) -> expression.Expression: """Return the standalone layers ``expression`` view of a corpus record.""" view, _complement = RECORD_EXPRESSION.forward(record) return view diff --git a/bead/interop/layers/graph_lens.py b/bead/interop/layers/graph_lens.py index 6f861f2..4d0e115 100644 --- a/bead/interop/layers/graph_lens.py +++ b/bead/interop/layers/graph_lens.py @@ -1,23 +1,28 @@ """Lens between a ``CorpusGraph`` and the layers property graph. -The lens projects a :class:`~bead.corpus.graph.CorpusGraph` to a layers-shaped -view (expression records, graph nodes, and a ``graphEdgeSet``) and keeps a -complement holding the information layers' graph does not express directly (the -bead framework identity, edge directedness, and exact float confidence). +The lens projects a :class:`~bead.corpus.graph.CorpusGraph` to a +:class:`CorpusGraphLayers` view (a bundle of canonical +:class:`lairs.records.expression.Expression` and +:class:`lairs.records.graph.GraphNode` records plus a +:class:`lairs.records.graph.GraphEdgeSet`) and keeps a complement holding the +information layers' graph does not express directly (the bead framework +identity, the bead node ids, edge directedness, and exact float confidence). Together the view and complement reconstruct the graph exactly. """ from __future__ import annotations import didactic.api as dx +from lairs.records import defs, expression, graph from bead.corpus.graph import CorpusEdge, CorpusGraph, CorpusNode from bead.corpus.records import CorpusRecord from bead.data.base import JsonValue from bead.interop.layers._convert import ( + CONFIDENCE_SCALE, apply_identity, - from_feature_map, - from_feature_map_scalar, + dumps_meta, + feature_map, from_object_ref, identity_of, j_bool, @@ -27,76 +32,92 @@ j_obj, j_str, j_str_or_none, + loads_meta, object_ref, - to_feature_map, + read_feature_map, + read_feature_map_scalar, ) -_CONFIDENCE_SCALE = 1000 +class CorpusGraphLayers(dx.Model): + """A layers view of a corpus graph: expressions, graph nodes, and an edge set. -class CorpusGraphLayersLens(dx.Lens[CorpusGraph, JsonValue, JsonValue]): + The records are ordered tuples; the bead node ids that key them live in the + lens complement (``layers`` records carry no bead node id). An empty graph + projects to empty tuples and an empty edge set. + """ + + expressions: tuple[dx.Embed[expression.Expression], ...] = dx.field(default=()) + graph_nodes: tuple[dx.Embed[graph.GraphNode], ...] = dx.field(default=()) + edge_set: dx.Embed[graph.GraphEdgeSet] = dx.field() + + +class CorpusGraphLayersLens(dx.Lens[CorpusGraph, CorpusGraphLayers, JsonValue]): """Lossless lens ``CorpusGraph <-> (layers graph view, bead complement)``.""" - def forward(self, graph: CorpusGraph) -> tuple[JsonValue, JsonValue]: + def forward(self, graph_in: CorpusGraph) -> tuple[CorpusGraphLayers, JsonValue]: """Project a graph to its layers view and bead complement.""" - expressions: dict[str, JsonValue] = {} - graph_nodes: dict[str, JsonValue] = {} + expressions: list[expression.Expression] = [] + graph_nodes: list[graph.GraphNode] = [] node_complements: dict[str, JsonValue] = {} node_order: list[JsonValue] = [] - for node in graph.nodes: + for node in graph_in.nodes: node_order.append(node.node_id) if node.record is not None: - expr: dict[str, JsonValue] = { - "id": node.node_id, - "kind": node.node_type, - "text": node.record.text, - "features": to_feature_map(node.record.provenance), - "createdAt": node.record.created_at.isoformat(), - } - if node.node_type_uri is not None: - expr["kindUri"] = node.node_type_uri - expressions[node.node_id] = expr + expressions.append( + expression.Expression( + id=node.node_id, + kind=node.node_type, + kindUri=node.node_type_uri, + text=node.record.text, + features=feature_map(node.record.provenance), + createdAt=node.record.created_at, + ) + ) node_complements[node.node_id] = { "is_expression": True, "identity": identity_of(node), "label": node.label, - "properties": to_feature_map(node.properties), + "properties": dumps_meta(node.properties), "record_identity": identity_of(node.record), "record_source_name": node.record.source_name, "record_index": node.record.record_index, } else: - graph_node: dict[str, JsonValue] = { - "nodeType": node.node_type, - "properties": to_feature_map(node.properties), - "createdAt": node.created_at.isoformat(), - } - if node.node_type_uri is not None: - graph_node["nodeTypeUri"] = node.node_type_uri - if node.label is not None: - graph_node["label"] = node.label - graph_nodes[node.node_id] = graph_node + graph_nodes.append( + graph.GraphNode( + nodeType=node.node_type, + nodeTypeUri=node.node_type_uri, + label=node.label, + properties=feature_map(node.properties), + createdAt=node.created_at, + ) + ) node_complements[node.node_id] = { "is_expression": False, "identity": identity_of(node), } - edge_views: list[JsonValue] = [] + edge_entries: list[graph.GraphEdgeEntry] = [] edge_complements: list[JsonValue] = [] - for edge in graph.edges: - edge_view: dict[str, JsonValue] = { - "edgeType": edge.edge_type, - "source": object_ref(edge.source_id), - "target": object_ref(edge.target_id), - "features": to_feature_map(edge.features), - } - if edge.edge_type_uri is not None: - edge_view["edgeTypeUri"] = edge.edge_type_uri - if edge.confidence is not None: - edge_view["confidence"] = round(edge.confidence * _CONFIDENCE_SCALE) - edge_view["uuid"] = {"value": str(edge.id)} - edge_views.append(edge_view) + for edge in graph_in.edges: + confidence = ( + round(edge.confidence * CONFIDENCE_SCALE) + if edge.confidence is not None + else None + ) + edge_entries.append( + graph.GraphEdgeEntry( + uuid=defs.Uuid(value=str(edge.id)), + edgeType=edge.edge_type, + edgeTypeUri=edge.edge_type_uri, + source=object_ref(edge.source_id), + target=object_ref(edge.target_id), + confidence=confidence, + features=feature_map(edge.features), + ) + ) edge_complements.append( { "identity": identity_of(edge), @@ -105,85 +126,84 @@ def forward(self, graph: CorpusGraph) -> tuple[JsonValue, JsonValue]: } ) - view: JsonValue = { - "expressions": expressions, - "graphNodes": graph_nodes, - "graphEdgeSet": { - "edges": tuple(edge_views), - "createdAt": graph.created_at.isoformat(), - }, - } + view = CorpusGraphLayers( + expressions=tuple(expressions), + graph_nodes=tuple(graph_nodes), + edge_set=graph.GraphEdgeSet( + createdAt=graph_in.created_at, edges=tuple(edge_entries) + ), + ) complement: JsonValue = { - "graph_identity": identity_of(graph), - "graph_metadata": to_feature_map(graph.graph_metadata), + "graph_identity": identity_of(graph_in), + "graph_metadata": dumps_meta(graph_in.graph_metadata), "node_order": tuple(node_order), "node_complements": node_complements, "edge_complements": tuple(edge_complements), } return view, complement - def backward(self, view: JsonValue, complement: JsonValue) -> CorpusGraph: + def backward( + self, view: CorpusGraphLayers, complement: JsonValue + ) -> CorpusGraph: """Reconstruct the graph from its layers view and bead complement.""" - view_obj = j_obj(view) comp = j_obj(complement) - expressions = j_obj(view_obj["expressions"]) - graph_nodes = j_obj(view_obj["graphNodes"]) node_complements = j_obj(comp["node_complements"]) nodes: list[CorpusNode] = [] + expr_index = 0 + graph_node_index = 0 for node_id_value in j_list(comp["node_order"]): node_id = j_str(node_id_value) node_comp = j_obj(node_complements[node_id]) if j_bool(node_comp["is_expression"]): - entry = j_obj(expressions[node_id]) + entry = view.expressions[expr_index] + expr_index += 1 record = apply_identity( CorpusRecord( - text=j_str(entry["text"]), + text=entry.text if entry.text is not None else "", source_name=j_str(node_comp["record_source_name"]), record_index=j_int(node_comp["record_index"]), - provenance=from_feature_map_scalar(entry["features"]), + provenance=read_feature_map_scalar(entry.features), ), node_comp["record_identity"], ) node = CorpusNode( node_id=node_id, - node_type=j_str(entry["kind"]), - node_type_uri=j_str_or_none(entry.get("kindUri")), + node_type=entry.kind, + node_type_uri=entry.kindUri, label=j_str_or_none(node_comp["label"]), record=record, - properties=from_feature_map(node_comp["properties"]), + properties=loads_meta(node_comp["properties"]), ) else: - entry = j_obj(graph_nodes[node_id]) + gnode = view.graph_nodes[graph_node_index] + graph_node_index += 1 node = CorpusNode( node_id=node_id, - node_type=j_str(entry["nodeType"]), - node_type_uri=j_str_or_none(entry.get("nodeTypeUri")), - label=j_str_or_none(entry.get("label")), + node_type=gnode.nodeType, + node_type_uri=gnode.nodeTypeUri, + label=gnode.label, record=None, - properties=from_feature_map(entry["properties"]), + properties=read_feature_map(gnode.properties), ) nodes.append(apply_identity(node, node_comp["identity"])) - edge_set = j_obj(view_obj["graphEdgeSet"]) - edge_views = j_list(edge_set["edges"]) edge_complements = j_list(comp["edge_complements"]) edges: list[CorpusEdge] = [] - for edge_view_value, edge_comp_value in zip( - edge_views, edge_complements, strict=True + for entry, edge_comp_value in zip( + view.edge_set.edges, edge_complements, strict=True ): - edge_view = j_obj(edge_view_value) edge_comp = j_obj(edge_comp_value) edges.append( apply_identity( CorpusEdge( - source_id=from_object_ref(edge_view["source"]), - target_id=from_object_ref(edge_view["target"]), - edge_type=j_str(edge_view["edgeType"]), - edge_type_uri=j_str_or_none(edge_view.get("edgeTypeUri")), + source_id=from_object_ref(entry.source), + target_id=from_object_ref(entry.target), + edge_type=entry.edgeType, + edge_type_uri=entry.edgeTypeUri, directed=j_bool(edge_comp["directed"]), confidence=j_float_or_none(edge_comp["confidence"]), - features=from_feature_map(edge_view["features"]), + features=read_feature_map(entry.features), ), edge_comp["identity"], ) @@ -193,7 +213,7 @@ def backward(self, view: JsonValue, complement: JsonValue) -> CorpusGraph: CorpusGraph( nodes=tuple(nodes), edges=tuple(edges), - graph_metadata=from_feature_map(comp["graph_metadata"]), + graph_metadata=loads_meta(comp["graph_metadata"]), ), comp["graph_identity"], ) @@ -202,7 +222,7 @@ def backward(self, view: JsonValue, complement: JsonValue) -> CorpusGraph: CORPUS_GRAPH_LAYERS = CorpusGraphLayersLens() -def graph_to_layers(graph: CorpusGraph) -> JsonValue: +def graph_to_layers(graph_in: CorpusGraph) -> CorpusGraphLayers: """Return the standalone layers-shaped view of a corpus graph.""" - view, _complement = CORPUS_GRAPH_LAYERS.forward(graph) + view, _complement = CORPUS_GRAPH_LAYERS.forward(graph_in) return view diff --git a/bead/interop/layers/model_lenses.py b/bead/interop/layers/model_lenses.py deleted file mode 100644 index d26182a..0000000 --- a/bead/interop/layers/model_lenses.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Generic lossless isos between faithful mirror models and layers JSON. - -A single :class:`MirrorIso` (parameterized by model type) serves every faithful -mirror model, since they all serialize through the structural snake<->camel -conversion in :mod:`bead.interop.layers._mirror`. ``SHARED_DEF_ISOS`` registers -one iso per shared-def mirror so a coverage test can assert every construct has -a law-passing mapping. -""" - -from __future__ import annotations - -import didactic.api as dx - -from bead.data.base import JsonValue -from bead.interop.layers._mirror import mirror_from_layers, mirror_to_layers -from bead.interop.layers.models import ( - AgentRef, - AlignmentLink, - Anchor, - AnnotationMetadata, - BoundingBox, - ExternalTarget, - Feature, - FeatureMap, - FragmentSelector, - Keyframe, - KnowledgeRef, - LayersConstraint, - LayersSpan, - LayersUuid, - ObjectRef, - PageAnchor, - Selector, - SpatialEntity, - SpatialExpression, - SpatialModifier, - SpatioTemporalAnchor, - TemporalEntity, - TemporalExpression, - TemporalModifier, - TemporalSpan, - TextPositionSelector, - TextQuoteSelector, - TokenRef, - TokenRefSequence, -) -from bead.interop.layers.models_records import ( - Annotation, - AnnotationLayer, - ArgumentRef, - AudioInfo, - Cluster, - DocumentInfo, - Expression, - GraphEdge, - GraphEdgeEntry, - GraphEdgeSet, - GraphNode, - RoleSlot, - Token, - Tokenization, - TypeDef, - VideoInfo, -) - - -class MirrorIso[T: dx.Model](dx.Iso[T, JsonValue]): - """Lossless iso between a faithful mirror model and layers-shaped JSON.""" - - def __init__(self, model_type: type[T]) -> None: - self._model_type = model_type - - def forward(self, model: T) -> JsonValue: - """Serialize the mirror model to layers JSON.""" - return mirror_to_layers(model) - - def backward(self, data: JsonValue) -> T: - """Deserialize layers JSON back into the mirror model.""" - return mirror_from_layers(self._model_type, data) - - -def mirror_iso[T: dx.Model](model_type: type[T]) -> MirrorIso[T]: - """Build a :class:`MirrorIso` for a mirror model type.""" - return MirrorIso(model_type) - - -#: Every shared-def mirror model, for coverage and registration. -SHARED_DEF_MODELS: tuple[type[dx.Model], ...] = ( - LayersUuid, - Feature, - FeatureMap, - KnowledgeRef, - BoundingBox, - TemporalSpan, - AgentRef, - ObjectRef, - LayersSpan, - TokenRef, - TokenRefSequence, - Keyframe, - SpatioTemporalAnchor, - TemporalEntity, - TemporalModifier, - TemporalExpression, - SpatialEntity, - SpatialModifier, - SpatialExpression, - PageAnchor, - TextQuoteSelector, - TextPositionSelector, - FragmentSelector, - Selector, - ExternalTarget, - Anchor, - AlignmentLink, - AnnotationMetadata, - LayersConstraint, -) - -#: One lossless iso per shared-def mirror model. -SHARED_DEF_ISOS: dict[type[dx.Model], MirrorIso[dx.Model]] = { - model_type: MirrorIso(model_type) for model_type in SHARED_DEF_MODELS -} - -#: Every linguistic record mirror model. -RECORD_MODELS: tuple[type[dx.Model], ...] = ( - Expression, - Token, - Tokenization, - ArgumentRef, - Annotation, - Cluster, - AnnotationLayer, - GraphNode, - GraphEdge, - GraphEdgeEntry, - GraphEdgeSet, - AudioInfo, - VideoInfo, - DocumentInfo, - RoleSlot, - TypeDef, -) - -#: One lossless iso per record mirror model. -RECORD_ISOS: dict[type[dx.Model], MirrorIso[dx.Model]] = { - model_type: MirrorIso(model_type) for model_type in RECORD_MODELS -} - -#: All mirror isos (shared defs + records), keyed by model type. -ALL_MIRROR_ISOS: dict[type[dx.Model], MirrorIso[dx.Model]] = { - **SHARED_DEF_ISOS, - **RECORD_ISOS, -} diff --git a/bead/interop/layers/models.py b/bead/interop/layers/models.py deleted file mode 100644 index 1a7ff6f..0000000 --- a/bead/interop/layers/models.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Faithful didactic mirrors of the ``layers`` shared object definitions. - -Each model mirrors a ``pub.layers.defs`` object field-for-field (snake_case -names corresponding to layers' camelCase, nested objects as embedded models, -feature maps as :class:`FeatureMap`, confidence as an integer 0-1000). The -structural fidelity lets :mod:`bead.interop.layers._mirror` serialize any of -them to and from layers JSON losslessly with a single generic conversion. - -Names that would clash with bead's own models are prefixed ``Layers``. -""" - -from __future__ import annotations - -import didactic.api as dx - - -class LayersUuid(dx.Model): - """A layers ``uuid`` value object.""" - - value: str - - -class Feature(dx.Model): - """A single key/value entry in a layers ``featureMap``.""" - - key: str - value: str - - -class FeatureMap(dx.Model): - """A layers ``featureMap`` (ordered key/value entries).""" - - entries: tuple[dx.Embed[Feature], ...] = () - - -class KnowledgeRef(dx.Model): - """A layers ``knowledgeRef`` grounding to an external knowledge base.""" - - source: str - identifier: str - source_uri: str | None = None - uri: str | None = None - label: str | None = None - - -class BoundingBox(dx.Model): - """A layers ``boundingBox`` (pixel region).""" - - x: int - y: int - width: int - height: int - - -class TemporalSpan(dx.Model): - """A layers ``temporalSpan`` (millisecond interval).""" - - start: int - ending: int - - -class AgentRef(dx.Model): - """A layers ``agentRef`` (annotating agent).""" - - did: str | None = None - id: str | None = None - name: str | None = None - knowledge_ref: dx.Embed[KnowledgeRef] | None = None - - -class ObjectRef(dx.Model): - """A layers ``objectRef`` (local, record, or external reference).""" - - local_id: dx.Embed[LayersUuid] | None = None - record_ref: str | None = None - object_id: dx.Embed[LayersUuid] | None = None - knowledge_ref: dx.Embed[KnowledgeRef] | None = None - - -class LayersSpan(dx.Model): - """A layers ``span`` (UTF-8 byte offsets, optional char offsets).""" - - byte_start: int - byte_end: int - char_start: int | None = None - char_end: int | None = None - - -class TokenRef(dx.Model): - """A layers ``tokenRef`` (single token in a tokenization).""" - - tokenization_id: dx.Embed[LayersUuid] - token_index: int - - -class TokenRefSequence(dx.Model): - """A layers ``tokenRefSequence`` (ordered tokens, optional anchor).""" - - tokenization_id: dx.Embed[LayersUuid] - token_indexes: tuple[int, ...] = () - anchor_token_index: int | None = None - - -class Keyframe(dx.Model): - """A layers ``keyframe`` (a bounding box at a video time).""" - - time_ms: int - bbox: dx.Embed[BoundingBox] - features: dx.Embed[FeatureMap] | None = None - - -class SpatioTemporalAnchor(dx.Model): - """A layers ``spatioTemporalAnchor`` (time span plus keyframes).""" - - temporal_span: dx.Embed[TemporalSpan] - keyframes: tuple[dx.Embed[Keyframe], ...] = () - interpolation_uri: str | None = None - interpolation: str | None = None - - -class TemporalEntity(dx.Model): - """A layers ``temporalEntity`` (instant/interval/duration value).""" - - instant: str | None = None - interval_start: str | None = None - interval_end: str | None = None - duration: str | None = None - earliest: str | None = None - latest: str | None = None - granularity_uri: str | None = None - granularity: str | None = None - calendar_uri: str | None = None - calendar: str | None = None - recurrence: str | None = None - features: dx.Embed[FeatureMap] | None = None - - -class TemporalModifier(dx.Model): - """A layers ``temporalModifier``.""" - - mod_uri: str | None = None - mod: str | None = None - features: dx.Embed[FeatureMap] | None = None - - -class TemporalExpression(dx.Model): - """A layers ``temporalExpression``.""" - - type_uri: str | None = None - type: str | None = None - value: dx.Embed[TemporalEntity] | None = None - modifier: dx.Embed[TemporalModifier] | None = None - anchor_ref: dx.Embed[ObjectRef] | None = None - function_uri: str | None = None - function: str | None = None - features: dx.Embed[FeatureMap] | None = None - - -class SpatialEntity(dx.Model): - """A layers ``spatialEntity`` (geometry/region value).""" - - bbox: dx.Embed[BoundingBox] | None = None - geometry: str | None = None - type_uri: str | None = None - type: str | None = None - geometry_format_uri: str | None = None - geometry_format: str | None = None - crs_uri: str | None = None - crs: str | None = None - dimensions: int | None = None - uncertainty: str | None = None - features: dx.Embed[FeatureMap] | None = None - - -class SpatialModifier(dx.Model): - """A layers ``spatialModifier``.""" - - mod_uri: str | None = None - mod: str | None = None - features: dx.Embed[FeatureMap] | None = None - - -class SpatialExpression(dx.Model): - """A layers ``spatialExpression``.""" - - type_uri: str | None = None - type: str | None = None - value: dx.Embed[SpatialEntity] | None = None - modifier: dx.Embed[SpatialModifier] | None = None - anchor_ref: dx.Embed[ObjectRef] | None = None - function_uri: str | None = None - function: str | None = None - features: dx.Embed[FeatureMap] | None = None - - -class PageAnchor(dx.Model): - """A layers ``pageAnchor`` (a region on a document page).""" - - page: int - bounding_box: dx.Embed[BoundingBox] | None = None - text_span: dx.Embed[LayersSpan] | None = None - - -class TextQuoteSelector(dx.Model): - """A W3C-style ``textQuoteSelector``.""" - - exact: str - prefix: str | None = None - suffix: str | None = None - - -class TextPositionSelector(dx.Model): - """A W3C-style ``textPositionSelector``.""" - - byte_start: int - byte_end: int - char_start: int | None = None - char_end: int | None = None - - -class FragmentSelector(dx.Model): - """A W3C-style ``fragmentSelector``.""" - - value: str - conforms_to: str | None = None - - -class Selector(dx.Model): - """The selector union of a layers ``externalTarget``.""" - - text_quote_selector: dx.Embed[TextQuoteSelector] | None = None - text_position_selector: dx.Embed[TextPositionSelector] | None = None - fragment_selector: dx.Embed[FragmentSelector] | None = None - - -class ExternalTarget(dx.Model): - """A layers ``externalTarget`` (a web resource + selector).""" - - source: str - source_hash: str | None = None - title: str | None = None - selector: dx.Embed[Selector] | None = None - - -class Anchor(dx.Model): - """A layers ``anchor`` (the polymorphic attachment point).""" - - text_span: dx.Embed[LayersSpan] | None = None - token_ref: dx.Embed[TokenRef] | None = None - token_ref_sequence: dx.Embed[TokenRefSequence] | None = None - temporal_span: dx.Embed[TemporalSpan] | None = None - spatio_temporal_anchor: dx.Embed[SpatioTemporalAnchor] | None = None - page_anchor: dx.Embed[PageAnchor] | None = None - external_target: dx.Embed[ExternalTarget] | None = None - - -class AlignmentLink(dx.Model): - """A layers ``alignmentLink`` (aligned token-index sets).""" - - source_indices: tuple[int, ...] = () - target_indices: tuple[int, ...] = () - confidence: int | None = None - label: str | None = None - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - features: dx.Embed[FeatureMap] | None = None - - -class AnnotationMetadata(dx.Model): - """A layers ``annotationMetadata`` (provenance for an annotation).""" - - tool: str - agent: dx.Embed[AgentRef] | None = None - timestamp: str | None = None - confidence: int | None = None - persona_ref: str | None = None - dependencies: tuple[dx.Embed[ObjectRef], ...] = () - digest: str | None = None - - -class LayersConstraint(dx.Model): - """A layers ``constraint`` (an expression with scope).""" - - expression: str - expression_format_uri: str | None = None - expression_format: str | None = None - scope_uri: str | None = None - scope: str | None = None - context: tuple[str, ...] = () - description: str | None = None diff --git a/bead/interop/layers/models_records.py b/bead/interop/layers/models_records.py deleted file mode 100644 index 0268af7..0000000 --- a/bead/interop/layers/models_records.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Faithful didactic mirrors of the linguistic ``layers`` record types. - -Mirrors the expression, segmentation, annotation, graph, media, and ontology -records field-for-field (reusing the shared-def mirrors in -:mod:`bead.interop.layers.models`). Like the shared defs, they serialize to and -from layers JSON losslessly through the generic snake<->camel conversion. -Binary ``blob`` fields are mirrored as their reference string. -""" - -from __future__ import annotations - -import didactic.api as dx - -from bead.interop.layers.models import ( - Anchor, - AnnotationMetadata, - FeatureMap, - KnowledgeRef, - LayersConstraint, - LayersSpan, - LayersUuid, - ObjectRef, - SpatialExpression, - TemporalExpression, - TemporalSpan, -) - - -class Expression(dx.Model): - """A layers ``expression`` (a text/linguistic unit, recursively nested).""" - - id: str - kind: str - created_at: str - kind_uri: str | None = None - text: str | None = None - parent_ref: str | None = None - anchor: dx.Embed[Anchor] | None = None - media_ref: str | None = None - media_blob: str | None = None - metadata: dx.Embed[AnnotationMetadata] | None = None - features: dx.Embed[FeatureMap] | None = None - source_url: str | None = None - source_ref: str | None = None - eprint_ref: str | None = None - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - languages: tuple[str, ...] = () - - -class Token(dx.Model): - """A layers ``token`` within a tokenization.""" - - token_index: int - text: str | None = None - text_span: dx.Embed[LayersSpan] | None = None - temporal_span: dx.Embed[TemporalSpan] | None = None - - -class Tokenization(dx.Model): - """A layers ``tokenization`` (one segmentation of an expression).""" - - uuid: dx.Embed[LayersUuid] - kind: str - kind_uri: str | None = None - expression_ref: str | None = None - tokens: tuple[dx.Embed[Token], ...] = () - metadata: dx.Embed[AnnotationMetadata] | None = None - - -class ArgumentRef(dx.Model): - """A layers ``argumentRef`` (a role-filling argument of a predicate).""" - - role: str - target: dx.Embed[ObjectRef] - features: dx.Embed[FeatureMap] | None = None - - -class Annotation(dx.Model): - """A layers ``annotation`` (the polymorphic annotation object).""" - - uuid: dx.Embed[LayersUuid] - anchor: dx.Embed[Anchor] | None = None - token_index: int | None = None - label: str | None = None - value: str | None = None - text: str | None = None - parent_id: dx.Embed[LayersUuid] | None = None - child_ids: tuple[dx.Embed[LayersUuid], ...] = () - head_index: int | None = None - target_index: int | None = None - arguments: tuple[dx.Embed[ArgumentRef], ...] = () - confidence: int | None = None - ontology_type_ref: str | None = None - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - temporal: dx.Embed[TemporalExpression] | None = None - spatial: dx.Embed[SpatialExpression] | None = None - features: dx.Embed[FeatureMap] | None = None - - -class Cluster(dx.Model): - """A layers ``cluster`` (a coreference/equivalence set).""" - - uuid: dx.Embed[LayersUuid] - canonical_label: str | None = None - members: tuple[dx.Embed[ObjectRef], ...] = () - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - features: dx.Embed[FeatureMap] | None = None - - -class AnnotationLayer(dx.Model): - """A layers ``annotationLayer`` (a typed layer of annotations).""" - - expression: str - kind: str - created_at: str - kind_uri: str | None = None - subkind_uri: str | None = None - subkind: str | None = None - formalism_uri: str | None = None - formalism: str | None = None - source_method_uri: str | None = None - source_method: str | None = None - label_set: str | None = None - ontology_ref: str | None = None - tokenization_id: dx.Embed[LayersUuid] | None = None - rank: int | None = None - alternatives_ref: str | None = None - parent_layer_ref: str | None = None - annotations: tuple[dx.Embed[Annotation], ...] = () - metadata: dx.Embed[AnnotationMetadata] | None = None - languages: tuple[str, ...] = () - - -class GraphNode(dx.Model): - """A layers ``graphNode`` (a standalone property-graph node).""" - - node_type: str - created_at: str - node_type_uri: str | None = None - label: str | None = None - properties: dx.Embed[FeatureMap] | None = None - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - metadata: dx.Embed[AnnotationMetadata] | None = None - - -class GraphEdge(dx.Model): - """A layers ``graphEdge`` (a single typed, directed edge record).""" - - source: dx.Embed[ObjectRef] - target: dx.Embed[ObjectRef] - edge_type: str - created_at: str - edge_type_uri: str | None = None - label: str | None = None - ordinal: int | None = None - confidence: int | None = None - properties: dx.Embed[FeatureMap] | None = None - metadata: dx.Embed[AnnotationMetadata] | None = None - - -class GraphEdgeEntry(dx.Model): - """A layers ``graphEdgeEntry`` (one edge within a graphEdgeSet).""" - - uuid: dx.Embed[LayersUuid] - edge_type: str - source: dx.Embed[ObjectRef] - target: dx.Embed[ObjectRef] - edge_type_uri: str | None = None - confidence: int | None = None - features: dx.Embed[FeatureMap] | None = None - - -class GraphEdgeSet(dx.Model): - """A layers ``graphEdgeSet`` (a batch of typed, directed edges).""" - - created_at: str - edges: tuple[dx.Embed[GraphEdgeEntry], ...] = () - expression: str | None = None - edge_type_uri: str | None = None - edge_type: str | None = None - metadata: dx.Embed[AnnotationMetadata] | None = None - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - features: dx.Embed[FeatureMap] | None = None - - -class AudioInfo(dx.Model): - """A layers ``audioInfo`` media descriptor.""" - - sample_rate: int | None = None - channels: int | None = None - bit_depth: int | None = None - codec: str | None = None - bit_rate: int | None = None - bit_rate_mode: str | None = None - number_of_samples: int | None = None - speaker_count: int | None = None - transcript_ref: str | None = None - segmentation_ref: str | None = None - - -class VideoInfo(dx.Model): - """A layers ``videoInfo`` media descriptor.""" - - width: int | None = None - height: int | None = None - frame_rate: int | None = None - codec: str | None = None - aspect_ratio: str | None = None - color_space: str | None = None - bit_rate: int | None = None - scan_type: str | None = None - - -class DocumentInfo(dx.Model): - """A layers ``documentInfo`` media descriptor.""" - - dpi: int | None = None - color_mode: str | None = None - page_count: int | None = None - script_system: str | None = None - writing_direction: str | None = None - ocr_engine: str | None = None - - -class RoleSlot(dx.Model): - """A layers ``roleSlot`` (a role in a type definition).""" - - role_name: str - role_description: str | None = None - filler_type_refs: tuple[str, ...] = () - collection_ref: str | None = None - required: bool | None = None - default_value: str | None = None - constraints: tuple[dx.Embed[LayersConstraint], ...] = () - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - features: dx.Embed[FeatureMap] | None = None - - -class TypeDef(dx.Model): - """A layers ``typeDef`` (an ontology type definition).""" - - ontology_ref: str - name: str - created_at: str - type_kind: str | None = None - type_kind_uri: str | None = None - gloss: str | None = None - parent_type_ref: str | None = None - allowed_roles: tuple[dx.Embed[RoleSlot], ...] = () - allowed_values: tuple[str, ...] = () - knowledge_refs: tuple[dx.Embed[KnowledgeRef], ...] = () - features: dx.Embed[FeatureMap] | None = None diff --git a/bead/interop/layers/parse_lens.py b/bead/interop/layers/parse_lens.py index e2c3e83..1772a89 100644 --- a/bead/interop/layers/parse_lens.py +++ b/bead/interop/layers/parse_lens.py @@ -1,28 +1,28 @@ """Lossless iso between a dependency parse and layers annotation records. -A :class:`~bead.tokenization.parsers.ParsedSentence` maps to a layers -``tokenization`` plus two annotation layers (a part-of-speech ``token-tag`` -layer and a ``dependency`` ``relation`` layer). ``ParsedToken``/``ParsedSentence`` -carry no framework identity, so the mapping is a true bijection (``dx.Iso``): -the layers view captures everything and reconstructs the parse exactly. +A :class:`~bead.tokenization.parsers.ParsedSentence` maps to a +:class:`ParsedSentenceLayers` view: a canonical +:class:`lairs.records.segmentation.Tokenization` plus two +:class:`lairs.records.annotation.AnnotationLayer` records (a part-of-speech +``token-tag`` layer and a dependency ``relation`` layer). +``ParsedToken``/``ParsedSentence`` carry no framework identity, so the mapping is +a true bijection (``dx.Iso``). + +The layers ``token`` has no ``spaceAfter`` slot, so each token's space-after flag +travels in its part-of-speech annotation's features alongside the lemma, xpos, +and morphology. The annotation layers require an expression reference and a +timestamp that the parse does not carry; these are emitted as fixed constants and +ignored on the way back. """ from __future__ import annotations +from datetime import UTC, datetime + import didactic.api as dx +from lairs.records import annotation, defs, segmentation -from bead.data.base import JsonValue -from bead.interop.layers._convert import ( - from_feature_map, - j_bool, - j_int, - j_list, - j_obj, - j_str, - j_str_or_none, - strip_nulls, - to_feature_map, -) +from bead.interop.layers._convert import feature_map, read_feature_map from bead.tokenization.parsers import ( UNIVERSAL_DEPENDENCIES, ParsedSentence, @@ -30,132 +30,149 @@ ) _ROOT_HEAD = -1 +_TOKENIZATION_UUID = "tokenization" +# The annotation layers require an expression reference and timestamp that a bare +# parse does not carry; these fixed constants are emitted forward and ignored +# backward, so the bijection holds. +_PARSE_EXPRESSION_URI = "at://local/parse" +_PARSE_CREATED_AT = datetime(1970, 1, 1, tzinfo=UTC) -def _opt_str(value: JsonValue) -> str | None: +def _opt_str(value: object) -> str | None: if value is None or isinstance(value, str): return value return str(value) -class ParsedSentenceLayersIso(dx.Iso[ParsedSentence, JsonValue]): +def _char(value: int | None) -> int: + """Return a character offset, defaulting absent offsets to 0.""" + return value if value is not None else 0 + + +class ParsedSentenceLayers(dx.Model): + """A layers view of a parse: a tokenization plus pos and dependency layers.""" + + original_text: str = dx.field() + tokenization: dx.Embed[segmentation.Tokenization] = dx.field() + pos_layer: dx.Embed[annotation.AnnotationLayer] = dx.field() + dependency_layer: dx.Embed[annotation.AnnotationLayer] = dx.field() + + +class ParsedSentenceLayersIso(dx.Iso[ParsedSentence, ParsedSentenceLayers]): """Lossless ``ParsedSentence <-> layers tokenization + annotation layers``.""" - def forward(self, sentence: ParsedSentence) -> JsonValue: + def forward(self, sentence: ParsedSentence) -> ParsedSentenceLayers: """Project a parsed sentence to layers tokenization + annotations.""" text = sentence.original_text def _byte(char_index: int) -> int: return len(text[:char_index].encode("utf-8")) - token_views: tuple[JsonValue, ...] = tuple( - { - "tokenIndex": token.index, - "text": token.text, - "textSpan": { - "byteStart": _byte(token.start_char), - "byteEnd": _byte(token.end_char), - "charStart": token.start_char, - "charEnd": token.end_char, - }, - "spaceAfter": token.space_after, - } + tokens = tuple( + segmentation.Token( + tokenIndex=token.index, + text=token.text, + textSpan=defs.Span( + byteStart=_byte(token.start_char), + byteEnd=_byte(token.end_char), + charStart=token.start_char, + charEnd=token.end_char, + ), + ) for token in sentence.tokens ) - pos_annotations: tuple[JsonValue, ...] = tuple( - { - "uuid": {"value": f"pos-{token.index}"}, - "tokenIndex": token.index, - "label": token.upos, - "features": to_feature_map( + pos_annotations = tuple( + annotation.Annotation( + uuid=defs.Uuid(value=f"pos-{token.index}"), + tokenIndex=token.index, + label=token.upos, + features=feature_map( { "xpos": token.xpos, "lemma": token.lemma, "morph": dict(token.morph), + "spaceAfter": token.space_after, } ), - } + ) for token in sentence.tokens ) - dependency_annotations: tuple[JsonValue, ...] = tuple( - { - "uuid": {"value": f"dep-{token.index}"}, - "tokenIndex": token.index, - "label": token.deprel, - "headIndex": token.head if token.head is not None else _ROOT_HEAD, - } + dependency_annotations = tuple( + annotation.Annotation( + uuid=defs.Uuid(value=f"dep-{token.index}"), + tokenIndex=token.index, + label=token.deprel, + headIndex=token.head if token.head is not None else _ROOT_HEAD, + ) for token in sentence.tokens ) - return strip_nulls( - { - "originalText": sentence.original_text, - "tokenization": { - "uuid": {"value": "tokenization"}, - "kind": "parser", - "tokens": token_views, - }, - "posLayer": { - "kind": "token-tag", - "subkind": "pos", - "formalism": UNIVERSAL_DEPENDENCIES, - "annotations": pos_annotations, - }, - "dependencyLayer": { - "kind": "relation", - "subkind": "dependency", - "formalism": UNIVERSAL_DEPENDENCIES, - "annotations": dependency_annotations, - }, - } + tokenization = segmentation.Tokenization( + uuid=defs.Uuid(value=_TOKENIZATION_UUID), + kind="custom", + tokens=tokens, + ) + return ParsedSentenceLayers( + original_text=text, + tokenization=tokenization, + pos_layer=annotation.AnnotationLayer( + annotations=pos_annotations, + createdAt=_PARSE_CREATED_AT, + expression=_PARSE_EXPRESSION_URI, + kind="token-tag", + subkind="pos", + formalism=UNIVERSAL_DEPENDENCIES, + tokenizationId=defs.Uuid(value=_TOKENIZATION_UUID), + ), + dependency_layer=annotation.AnnotationLayer( + annotations=dependency_annotations, + createdAt=_PARSE_CREATED_AT, + expression=_PARSE_EXPRESSION_URI, + kind="relation", + subkind="dependency", + formalism=UNIVERSAL_DEPENDENCIES, + tokenizationId=defs.Uuid(value=_TOKENIZATION_UUID), + ), ) - def backward(self, view: JsonValue) -> ParsedSentence: + def backward(self, view: ParsedSentenceLayers) -> ParsedSentence: """Reconstruct a parsed sentence from its layers projection.""" - view_obj = j_obj(view) - tokenization = j_obj(view_obj["tokenization"]) - token_views = j_list(tokenization["tokens"]) - pos_annotations = j_list(j_obj(view_obj["posLayer"])["annotations"]) - dep_annotations = j_list(j_obj(view_obj["dependencyLayer"])["annotations"]) - tokens: list[ParsedToken] = [] - for token_value, pos_value, dep_value in zip( - token_views, pos_annotations, dep_annotations, strict=True + for token, pos, dep in zip( + view.tokenization.tokens, + view.pos_layer.annotations, + view.dependency_layer.annotations, + strict=True, ): - token_obj = j_obj(token_value) - pos_obj = j_obj(pos_value) - dep_obj = j_obj(dep_value) - span = j_obj(token_obj["textSpan"]) - features = from_feature_map(pos_obj["features"]) + features = read_feature_map(pos.features) raw_morph = features.get("morph") morph = ( {key: str(value) for key, value in raw_morph.items()} if isinstance(raw_morph, dict) else {} ) - head_index = j_int(dep_obj["headIndex"]) + head_index = dep.headIndex if dep.headIndex is not None else _ROOT_HEAD + span = token.textSpan tokens.append( ParsedToken( - index=j_int(token_obj["tokenIndex"]), - text=j_str(token_obj["text"]), + index=token.tokenIndex, + text=token.text if token.text is not None else "", lemma=_opt_str(features.get("lemma")), - upos=j_str_or_none(pos_obj.get("label")), + upos=pos.label, xpos=_opt_str(features.get("xpos")), - deprel=j_str_or_none(dep_obj.get("label")), + deprel=dep.label, head=None if head_index == _ROOT_HEAD else head_index, morph=morph, - space_after=j_bool(token_obj["spaceAfter"]), - start_char=j_int(span["charStart"]), - end_char=j_int(span["charEnd"]), + space_after=bool(features.get("spaceAfter")), + start_char=_char(span.charStart if span is not None else None), + end_char=_char(span.charEnd if span is not None else None), ) ) - return ParsedSentence( - original_text=j_str(view_obj["originalText"]), tokens=tuple(tokens) - ) + return ParsedSentence(original_text=view.original_text, tokens=tuple(tokens)) PARSED_SENTENCE_LAYERS = ParsedSentenceLayersIso() -def parse_to_layers(sentence: ParsedSentence) -> JsonValue: +def parse_to_layers(sentence: ParsedSentence) -> ParsedSentenceLayers: """Return the layers tokenization + annotation-layer view of a parse.""" return PARSED_SENTENCE_LAYERS.forward(sentence) diff --git a/bead/interop/layers/resource_lens.py b/bead/interop/layers/resource_lens.py index aa2a34c..107e1b5 100644 --- a/bead/interop/layers/resource_lens.py +++ b/bead/interop/layers/resource_lens.py @@ -1,32 +1,35 @@ """Lenses between bead resource models and layers resource records. -Maps bead's lexical and template resources to their ``layers`` counterparts: +Maps bead's lexical and template resources to their canonical +:mod:`lairs.records.resource` counterparts: - ``LexicalItem`` <-> a layers ``entry`` - ``Lexicon`` <-> a layers ``collection`` with its ``entry`` records - ``Template`` <-> a layers ``template`` (with its slots and constraints) -Each lens produces a layers-shaped view and keeps the fields that have no layers -equivalent (the bead framework identity, the single language code, tags, the -``LexicalItem`` ``form``/``source`` fields, and DSL constraint context) in the -lens complement, so reconstruction is exact. +Each lens produces a layers-shaped view from the generated models and keeps the +fields that have no layers equivalent (the bead framework identity, tags, the +``LexicalItem`` original ``form`` / free-text ``source``, and the bead DSL +constraint context) in the lens complement, so reconstruction is exact. """ from __future__ import annotations import didactic.api as dx +from lairs.records import defs, resource from bead.data.base import JsonValue from bead.interop.layers._convert import ( apply_identity, - from_feature_map, + dumps_meta, + feature_map, identity_of, - j_bool, j_list, j_obj, j_str, j_str_or_none, - to_feature_map, + loads_meta, + read_feature_map, ) from bead.resources.constraints import Constraint from bead.resources.lexical_item import LexicalItem @@ -36,35 +39,41 @@ _LEXICON_KIND = "lexicon" -class LexicalItemEntryLens(dx.Lens[LexicalItem, JsonValue, JsonValue]): - """Lossless lens ``LexicalItem <-> (layers entry view, complement)``.""" +def _languages(code: str | None) -> tuple[str, ...]: + return (code,) if code is not None else () - def forward(self, item: LexicalItem) -> tuple[JsonValue, JsonValue]: - """Project a lexical item to a layers entry view and complement.""" - view: dict[str, JsonValue] = { - "form": item.form if item.form is not None else item.lemma, - "lemma": item.lemma, - "features": to_feature_map(item.features), - } - if item.language_code is not None: - view["languages"] = (item.language_code,) + +def _first_language(languages: tuple[str, ...] | None) -> str | None: + return languages[0] if languages else None + + +class LexicalItemEntryLens(dx.Lens[LexicalItem, resource.Entry, JsonValue]): + """Lossless lens ``LexicalItem <-> (layers entry, complement)``.""" + + def forward(self, item: LexicalItem) -> tuple[resource.Entry, JsonValue]: + """Project a lexical item to a layers entry and complement.""" + view = resource.Entry( + form=item.form if item.form is not None else item.lemma, + lemma=item.lemma, + features=feature_map(item.features), + languages=_languages(item.language_code), + createdAt=item.created_at, + ) complement: JsonValue = { "identity": identity_of(item), "form": item.form, - "language_code": item.language_code, "source": item.source, } return view, complement - def backward(self, view: JsonValue, complement: JsonValue) -> LexicalItem: - """Reconstruct a lexical item from its layers entry view and complement.""" - view_obj = j_obj(view) + def backward(self, view: resource.Entry, complement: JsonValue) -> LexicalItem: + """Reconstruct a lexical item from its layers entry and complement.""" comp = j_obj(complement) item = LexicalItem( - lemma=j_str(view_obj["lemma"]), - language_code=j_str_or_none(comp["language_code"]), + lemma=view.lemma if view.lemma is not None else "", + language_code=_first_language(view.languages), form=j_str_or_none(comp["form"]), - features=from_feature_map(view_obj["features"]), + features=read_feature_map(view.features), source=j_str_or_none(comp["source"]), ) return apply_identity(item, comp["identity"]) @@ -73,47 +82,55 @@ def backward(self, view: JsonValue, complement: JsonValue) -> LexicalItem: LEXICAL_ITEM_ENTRY = LexicalItemEntryLens() -class LexiconCollectionLens(dx.Lens[Lexicon, JsonValue, JsonValue]): +class LexiconLayers(dx.Model): + """A layers view of a lexicon: a collection plus its entry records.""" + + collection: dx.Embed[resource.Collection] = dx.field() + entries: tuple[dx.Embed[resource.Entry], ...] = dx.field(default=()) + + +class LexiconCollectionLens(dx.Lens[Lexicon, LexiconLayers, JsonValue]): """Lossless lens ``Lexicon <-> (layers collection + entries, complement)``.""" - def forward(self, lexicon: Lexicon) -> tuple[JsonValue, JsonValue]: + def forward(self, lexicon: Lexicon) -> tuple[LexiconLayers, JsonValue]: """Project a lexicon to a layers collection + entry views.""" - collection: dict[str, JsonValue] = {"name": lexicon.name, "kind": _LEXICON_KIND} - if lexicon.description is not None: - collection["description"] = lexicon.description - if lexicon.language_code is not None: - collection["languages"] = (lexicon.language_code,) - entries: list[JsonValue] = [] + entries: list[resource.Entry] = [] item_complements: list[JsonValue] = [] for item in lexicon.items: entry_view, entry_complement = LEXICAL_ITEM_ENTRY.forward(item) entries.append(entry_view) item_complements.append(entry_complement) - view: JsonValue = {"collection": collection, "entries": tuple(entries)} + view = LexiconLayers( + collection=resource.Collection( + name=lexicon.name, + kind=_LEXICON_KIND, + description=lexicon.description, + languages=_languages(lexicon.language_code), + createdAt=lexicon.created_at, + ), + entries=tuple(entries), + ) complement: JsonValue = { "identity": identity_of(lexicon), - "description": lexicon.description, - "language_code": lexicon.language_code, "tags": lexicon.tags, "item_complements": tuple(item_complements), } return view, complement - def backward(self, view: JsonValue, complement: JsonValue) -> Lexicon: + def backward(self, view: LexiconLayers, complement: JsonValue) -> Lexicon: """Reconstruct a lexicon from its layers collection + complement.""" - view_obj = j_obj(view) comp = j_obj(complement) - collection = j_obj(view_obj["collection"]) - entries = j_list(view_obj["entries"]) item_complements = j_list(comp["item_complements"]) items = tuple( LEXICAL_ITEM_ENTRY.backward(entry, item_complement) - for entry, item_complement in zip(entries, item_complements, strict=True) + for entry, item_complement in zip( + view.entries, item_complements, strict=True + ) ) lexicon = Lexicon( - name=j_str(collection["name"]), - description=j_str_or_none(comp["description"]), - language_code=j_str_or_none(comp["language_code"]), + name=view.collection.name, + description=view.collection.description, + language_code=_first_language(view.collection.languages), items=items, tags=tuple(j_str(tag) for tag in j_list(comp["tags"])), ) @@ -123,41 +140,41 @@ def backward(self, view: JsonValue, complement: JsonValue) -> Lexicon: LEXICON_COLLECTION = LexiconCollectionLens() -def _constraint_forward(constraint: Constraint) -> tuple[JsonValue, JsonValue]: - view: dict[str, JsonValue] = {"expression": constraint.expression} - if constraint.description is not None: - view["description"] = constraint.description +def _constraint_forward(constraint: Constraint) -> tuple[defs.Constraint, JsonValue]: + view = defs.Constraint( + expression=constraint.expression, description=constraint.description + ) complement: JsonValue = { "identity": identity_of(constraint), - "context": to_feature_map(constraint.context), + "context": dumps_meta(constraint.context), } return view, complement -def _constraint_backward(view: JsonValue, complement: JsonValue) -> Constraint: - view_obj = j_obj(view) +def _constraint_backward(view: defs.Constraint, complement: JsonValue) -> Constraint: comp = j_obj(complement) constraint = Constraint( - expression=j_str(view_obj["expression"]), - context=from_feature_map(comp["context"]), - description=j_str_or_none(view_obj.get("description")), + expression=view.expression, + context=loads_meta(comp["context"]), + description=view.description, ) return apply_identity(constraint, comp["identity"]) -def _slot_forward(slot: Slot) -> tuple[JsonValue, JsonValue]: - view: dict[str, JsonValue] = {"name": slot.name, "required": slot.required} - if slot.description is not None: - view["description"] = slot.description - if slot.default_value is not None: - view["defaultValue"] = slot.default_value - constraint_views: list[JsonValue] = [] +def _slot_forward(slot: Slot) -> tuple[resource.Slot, JsonValue]: + constraint_views: list[defs.Constraint] = [] constraint_complements: list[JsonValue] = [] for constraint in slot.constraints: constraint_view, constraint_complement = _constraint_forward(constraint) constraint_views.append(constraint_view) constraint_complements.append(constraint_complement) - view["constraints"] = tuple(constraint_views) + view = resource.Slot( + name=slot.name, + description=slot.description, + constraints=tuple(constraint_views), + defaultValue=slot.default_value, + required=slot.required, + ) complement: JsonValue = { "identity": identity_of(slot), "constraint_complements": tuple(constraint_complements), @@ -165,93 +182,88 @@ def _slot_forward(slot: Slot) -> tuple[JsonValue, JsonValue]: return view, complement -def _slot_backward(view: JsonValue, complement: JsonValue) -> Slot: - view_obj = j_obj(view) +def _slot_backward(view: resource.Slot, complement: JsonValue) -> Slot: comp = j_obj(complement) - constraint_views = j_list(view_obj["constraints"]) constraint_complements = j_list(comp["constraint_complements"]) constraints = tuple( _constraint_backward(constraint_view, constraint_complement) for constraint_view, constraint_complement in zip( - constraint_views, constraint_complements, strict=True + view.constraints or (), constraint_complements, strict=True ) ) slot = Slot( - name=j_str(view_obj["name"]), - description=j_str_or_none(view_obj.get("description")), + name=view.name, + description=view.description, constraints=constraints, - required=j_bool(view_obj["required"]), - default_value=j_str_or_none(view_obj.get("defaultValue")), + required=view.required if view.required is not None else True, + default_value=view.defaultValue, ) return apply_identity(slot, comp["identity"]) -class TemplateLayersLens(dx.Lens[Template, JsonValue, JsonValue]): - """Lossless lens ``Template <-> (layers template view, complement)``.""" +class TemplateLayersLens(dx.Lens[Template, resource.Template, JsonValue]): + """Lossless lens ``Template <-> (layers template, complement)``.""" - def forward(self, template: Template) -> tuple[JsonValue, JsonValue]: - """Project a template to a layers template view and complement.""" - slot_views: dict[str, JsonValue] = {} + def forward(self, template: Template) -> tuple[resource.Template, JsonValue]: + """Project a template to a layers template and complement.""" + slot_views: list[resource.Slot] = [] slot_complements: dict[str, JsonValue] = {} for slot_key, slot in template.slots.items(): slot_view, slot_complement = _slot_forward(slot) - slot_views[slot_key] = slot_view + slot_views.append(slot_view) slot_complements[slot_key] = slot_complement - constraint_views: list[JsonValue] = [] + constraint_views: list[defs.Constraint] = [] constraint_complements: list[JsonValue] = [] for constraint in template.constraints: constraint_view, constraint_complement = _constraint_forward(constraint) constraint_views.append(constraint_view) constraint_complements.append(constraint_complement) - view: dict[str, JsonValue] = { - "name": template.name, - "text": template.template_string, - "slots": slot_views, - "constraints": tuple(constraint_views), - } - if template.language_code is not None: - view["languages"] = (template.language_code,) + view = resource.Template( + name=template.name, + text=template.template_string, + slots=tuple(slot_views), + constraints=tuple(constraint_views), + languages=_languages(template.language_code), + createdAt=template.created_at, + ) complement: JsonValue = { "identity": identity_of(template), "description": template.description, - "language_code": template.language_code, "tags": template.tags, - "metadata": to_feature_map(template.metadata), + "metadata": dumps_meta(template.metadata), "slot_order": tuple(template.slots), "slot_complements": slot_complements, "constraint_complements": tuple(constraint_complements), } return view, complement - def backward(self, view: JsonValue, complement: JsonValue) -> Template: - """Reconstruct a template from its layers template view and complement.""" - view_obj = j_obj(view) + def backward( + self, view: resource.Template, complement: JsonValue + ) -> Template: + """Reconstruct a template from its layers template and complement.""" comp = j_obj(complement) - slot_views = j_obj(view_obj["slots"]) slot_complements = j_obj(comp["slot_complements"]) + slot_order = j_list(comp["slot_order"]) slots: dict[str, Slot] = {} - for slot_key_value in j_list(comp["slot_order"]): + for slot_key_value, slot_view in zip(slot_order, view.slots, strict=True): slot_key = j_str(slot_key_value) - slots[slot_key] = _slot_backward( - slot_views[slot_key], slot_complements[slot_key] - ) - constraint_views = j_list(view_obj["constraints"]) + slots[slot_key] = _slot_backward(slot_view, slot_complements[slot_key]) constraint_complements = j_list(comp["constraint_complements"]) constraints = tuple( _constraint_backward(constraint_view, constraint_complement) for constraint_view, constraint_complement in zip( - constraint_views, constraint_complements, strict=True + view.constraints or (), constraint_complements, strict=True ) ) template = Template( - name=j_str(view_obj["name"]), - template_string=j_str(view_obj["text"]), + name=view.name if view.name is not None else "", + template_string=view.text, slots=slots, constraints=constraints, description=j_str_or_none(comp["description"]), - language_code=j_str_or_none(comp["language_code"]), + language_code=_first_language(view.languages), tags=tuple(j_str(tag) for tag in j_list(comp["tags"])), - metadata=from_feature_map(comp["metadata"]), + metadata=loads_meta(comp["metadata"]), ) return apply_identity(template, comp["identity"]) diff --git a/tests/interop/lexicon_validator/.gitignore b/tests/interop/lexicon_validator/.gitignore deleted file mode 100644 index 504afef..0000000 --- a/tests/interop/lexicon_validator/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules/ -package-lock.json diff --git a/tests/interop/lexicon_validator/package.json b/tests/interop/lexicon_validator/package.json deleted file mode 100644 index dac3317..0000000 --- a/tests/interop/lexicon_validator/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "bead-layers-lexicon-validator", - "private": true, - "type": "module", - "description": "Validates bead's layers-mapping output against the layers lexicons using @atproto/lexicon.", - "dependencies": { - "@atproto/lexicon": "^0.5.0" - } -} diff --git a/tests/interop/lexicon_validator/validate.mjs b/tests/interop/lexicon_validator/validate.mjs deleted file mode 100644 index 92d4723..0000000 --- a/tests/interop/lexicon_validator/validate.mjs +++ /dev/null @@ -1,45 +0,0 @@ -// Validates JSON values against the layers lexicons using the ATProto lexicon -// validation machinery (@atproto/lexicon). Reads a JSON array of -// {lexUri, value} pairs from stdin and writes a JSON array of {ok, error?}. -import { readFileSync, readdirSync } from "fs"; -import { dirname, join } from "path"; -import { fileURLToPath } from "url"; -import { Lexicons } from "@atproto/lexicon"; - -const here = dirname(fileURLToPath(import.meta.url)); -const lexiconDir = - process.env.LAYERS_LEXICON_DIR || - join(here, "..", "..", "..", "vendor", "layers", "lexicons", "pub", "layers"); - -function lexiconFiles(dir) { - const out = []; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - const full = join(dir, entry.name); - if (entry.isDirectory()) out.push(...lexiconFiles(full)); - else if (entry.name.endsWith(".json")) out.push(full); - } - return out.sort(); -} - -const lexicons = new Lexicons(); -for (const file of lexiconFiles(lexiconDir)) { - lexicons.add(JSON.parse(readFileSync(file, "utf8"))); -} - -let input = ""; -process.stdin.setEncoding("utf8"); -process.stdin.on("data", (chunk) => (input += chunk)); -process.stdin.on("end", () => { - const items = JSON.parse(input); - const results = items.map(({ lexUri, value }) => { - try { - const result = lexicons.validate(lexUri, value); - return result.success - ? { ok: true } - : { ok: false, lexUri, error: result.error?.message ?? "invalid" }; - } catch (err) { - return { ok: false, lexUri, error: String(err?.message ?? err) }; - } - }); - process.stdout.write(JSON.stringify(results)); -}); diff --git a/tests/interop/test_layers_coverage.py b/tests/interop/test_layers_coverage.py deleted file mode 100644 index bd559d8..0000000 --- a/tests/interop/test_layers_coverage.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Coverage guard: every targeted layers construct has a law-passing mapping. - -If a new layers construct is mirrored, it must be registered (and round-trip -tested in the per-construct suites). This test fails loudly if a targeted -construct loses its registered iso. -""" - -from __future__ import annotations - -from bead.interop.layers.model_lenses import ALL_MIRROR_ISOS, MirrorIso - -# layers construct slug -> bead mirror model class name. -_EXPECTED: dict[str, str] = { - # pub.layers.defs shared objects - "uuid": "LayersUuid", - "feature": "Feature", - "featureMap": "FeatureMap", - "knowledgeRef": "KnowledgeRef", - "boundingBox": "BoundingBox", - "temporalSpan": "TemporalSpan", - "agentRef": "AgentRef", - "objectRef": "ObjectRef", - "span": "LayersSpan", - "tokenRef": "TokenRef", - "tokenRefSequence": "TokenRefSequence", - "keyframe": "Keyframe", - "spatioTemporalAnchor": "SpatioTemporalAnchor", - "temporalEntity": "TemporalEntity", - "temporalModifier": "TemporalModifier", - "temporalExpression": "TemporalExpression", - "spatialEntity": "SpatialEntity", - "spatialModifier": "SpatialModifier", - "spatialExpression": "SpatialExpression", - "pageAnchor": "PageAnchor", - "textQuoteSelector": "TextQuoteSelector", - "textPositionSelector": "TextPositionSelector", - "fragmentSelector": "FragmentSelector", - "externalTarget": "ExternalTarget", - "anchor": "Anchor", - "alignmentLink": "AlignmentLink", - "annotationMetadata": "AnnotationMetadata", - "constraint": "LayersConstraint", - # linguistic record types - "expression": "Expression", - "token": "Token", - "tokenization": "Tokenization", - "argumentRef": "ArgumentRef", - "annotation": "Annotation", - "cluster": "Cluster", - "annotationLayer": "AnnotationLayer", - "graphNode": "GraphNode", - "graphEdge": "GraphEdge", - "graphEdgeEntry": "GraphEdgeEntry", - "graphEdgeSet": "GraphEdgeSet", - "audioInfo": "AudioInfo", - "videoInfo": "VideoInfo", - "documentInfo": "DocumentInfo", - "roleSlot": "RoleSlot", - "typeDef": "TypeDef", -} - - -def test_every_targeted_construct_is_mapped() -> None: - mapped_model_names = {model_type.__name__ for model_type in ALL_MIRROR_ISOS} - missing = { - slug: name for slug, name in _EXPECTED.items() if name not in mapped_model_names - } - assert not missing, f"layers constructs without a mirror iso: {missing}" - - -def test_all_registrations_are_law_lenses() -> None: - assert ALL_MIRROR_ISOS - assert all(isinstance(iso, MirrorIso) for iso in ALL_MIRROR_ISOS.values()) diff --git a/tests/interop/test_layers_defs.py b/tests/interop/test_layers_defs.py deleted file mode 100644 index 27ef36f..0000000 --- a/tests/interop/test_layers_defs.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Round-trip law tests for the layers shared-def mirror isos.""" - -from __future__ import annotations - -import didactic.api as dx -import pytest -from didactic.lenses._testing import verify_iso -from hypothesis import strategies as st - -from bead.interop.layers import models as m -from bead.interop.layers.model_lenses import SHARED_DEF_ISOS, MirrorIso, mirror_iso - -_KR = m.KnowledgeRef(source="wikidata", identifier="Q5") -_UUID = m.LayersUuid(value="u1") -_BBOX = m.BoundingBox(x=1, y=2, width=3, height=4) -_FEATURES = m.FeatureMap(entries=(m.Feature(key="k", value="v"),)) - -# One representative instance per shared-def mirror model. -_EXAMPLES: tuple[dx.Model, ...] = ( - _UUID, - m.Feature(key="k", value="v"), - _FEATURES, - m.KnowledgeRef(source="wikidata", identifier="Q5", label="human"), - _BBOX, - m.TemporalSpan(start=0, ending=100), - m.AgentRef(did="did:plc:x", name="A", knowledge_ref=_KR), - m.ObjectRef(local_id=_UUID, knowledge_ref=_KR), - m.LayersSpan(byte_start=0, byte_end=5, char_start=0, char_end=5), - m.TokenRef(tokenization_id=_UUID, token_index=2), - m.TokenRefSequence( - tokenization_id=_UUID, token_indexes=(1, 2, 3), anchor_token_index=2 - ), - m.Keyframe(time_ms=10, bbox=_BBOX, features=_FEATURES), - m.SpatioTemporalAnchor( - temporal_span=m.TemporalSpan(start=0, ending=10), - keyframes=(m.Keyframe(time_ms=1, bbox=_BBOX),), - interpolation="linear", - ), - m.TemporalEntity(instant="2026-05-29", granularity="day", features=_FEATURES), - m.TemporalModifier(mod="approx"), - m.TemporalExpression( - type="date", - value=m.TemporalEntity(instant="2026-05-29"), - modifier=m.TemporalModifier(mod="approx"), - anchor_ref=m.ObjectRef(local_id=_UUID), - ), - m.SpatialEntity(geometry="POINT(0 0)", type="point", dimensions=2), - m.SpatialModifier(mod="near"), - m.SpatialExpression(type="loc", value=m.SpatialEntity(geometry="g")), - m.PageAnchor( - page=1, bounding_box=_BBOX, text_span=m.LayersSpan(byte_start=0, byte_end=2) - ), - m.TextQuoteSelector(exact="quote", prefix="a", suffix="b"), - m.TextPositionSelector(byte_start=0, byte_end=5), - m.FragmentSelector(value="#frag", conforms_to="https://example/spec"), - m.Selector(text_quote_selector=m.TextQuoteSelector(exact="q")), - m.ExternalTarget( - source="http://x", - title="t", - selector=m.Selector(fragment_selector=m.FragmentSelector(value="#f")), - ), - m.Anchor(token_ref=m.TokenRef(tokenization_id=_UUID, token_index=0)), - m.AlignmentLink( - source_indices=(0, 1), - target_indices=(2,), - confidence=900, - label="align", - knowledge_refs=(_KR,), - ), - m.AnnotationMetadata( - tool="spacy", - agent=m.AgentRef(name="A"), - timestamp="2026-05-29T00:00:00+00:00", - confidence=950, - dependencies=(m.ObjectRef(local_id=_UUID),), - ), - m.LayersConstraint( - expression="x>0", scope="token", context=("a", "b"), description="d" - ), -) - - -@pytest.mark.parametrize("example", _EXAMPLES, ids=lambda e: type(e).__name__) -def test_shared_def_roundtrip(example: dx.Model) -> None: - iso = SHARED_DEF_ISOS[type(example)] - view = iso.forward(example) - # GetPut: reconstruct exactly from the layers JSON. - assert iso.backward(view) == example - # PutGet: re-projection is stable. - assert iso.forward(iso.backward(view)) == view - - -def test_every_shared_def_has_a_law_passing_iso() -> None: - # Coverage guard: each example's type has a registered iso, and every - # registered iso is exercised by an example (no silent omission). - example_types = {type(example) for example in _EXAMPLES} - assert example_types == set(SHARED_DEF_ISOS) - - -def test_camelcase_projection() -> None: - view = mirror_iso(m.LayersSpan).forward( - m.LayersSpan(byte_start=1, byte_end=9, char_start=1, char_end=9) - ) - assert view == {"byteStart": 1, "byteEnd": 9, "charStart": 1, "charEnd": 9} - - -# --- didactic law verification on flat models ------------------------------- - - -def test_verify_iso_uuid() -> None: - iso: MirrorIso[m.LayersUuid] = mirror_iso(m.LayersUuid) - verify_iso(iso, st.builds(m.LayersUuid, value=st.text(max_size=8)), max_examples=30) - - -def test_verify_iso_bounding_box() -> None: - iso: MirrorIso[m.BoundingBox] = mirror_iso(m.BoundingBox) - ints = st.integers(0, 1000) - verify_iso( - iso, - st.builds(m.BoundingBox, x=ints, y=ints, width=ints, height=ints), - max_examples=30, - ) - - -def test_verify_iso_knowledge_ref() -> None: - iso: MirrorIso[m.KnowledgeRef] = mirror_iso(m.KnowledgeRef) - text = st.text(max_size=6) - opt = st.one_of(st.none(), text) - verify_iso( - iso, - st.builds( - m.KnowledgeRef, - source=text, - identifier=text, - source_uri=opt, - uri=opt, - label=opt, - ), - max_examples=30, - ) diff --git a/tests/interop/test_layers_graph_roundtrip.py b/tests/interop/test_layers_graph_roundtrip.py index 1e8a66e..fda1bdd 100644 --- a/tests/interop/test_layers_graph_roundtrip.py +++ b/tests/interop/test_layers_graph_roundtrip.py @@ -4,6 +4,7 @@ from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from lairs.records import defs from bead.corpus.assemble import EdgeSpec, assemble_graph from bead.corpus.graph import CorpusEdge, CorpusGraph, CorpusNode @@ -108,13 +109,12 @@ def test_view_is_layers_shaped(self) -> None: edges=(CorpusEdge(source_id="x", target_id="y", edge_type="e"),), ) view = graph_to_layers(graph) - assert set(view) == {"expressions", "graphNodes", "graphEdgeSet"} - edge = view["graphEdgeSet"]["edges"][0] - assert edge["edgeType"] == "e" - assert edge["source"] == {"localId": {"value": "x"}} - assert edge["target"] == {"localId": {"value": "y"}} - assert view["expressions"]["x"]["kind"] == "expression" - assert view["expressions"]["x"]["text"] == "t" + edge = view.edge_set.edges[0] + assert edge.edgeType == "e" + assert edge.source == defs.ObjectRef(localId=defs.Uuid(value="x")) + assert edge.target == defs.ObjectRef(localId=defs.Uuid(value="y")) + assert view.expressions[0].kind == "expression" + assert view.expressions[0].text == "t" # --- property-based lens-law verification ----------------------------------- diff --git a/tests/interop/test_layers_lexicon_validation.py b/tests/interop/test_layers_lexicon_validation.py deleted file mode 100644 index a68afdb..0000000 --- a/tests/interop/test_layers_lexicon_validation.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Validate bead's layers-mapping output against the layers lexicons. - -Uses the ATProto lexicon validation machinery (``@atproto/lexicon``) against the -layers lexicons vendored as the ``vendor/layers`` git submodule (checked out with -``git submodule update --init``) to prove every mapping produces schema-valid -layers records. The validator runs in Node; the suite skips if Node, the -validator dependency, or the submodule checkout is unavailable. -""" - -from __future__ import annotations - -import json -import shutil -import subprocess -from pathlib import Path - -import pytest - -from bead.corpus.graph import CorpusEdge, CorpusGraph, CorpusNode -from bead.corpus.records import CorpusRecord -from bead.interop.layers import models as m -from bead.interop.layers import models_records as r -from bead.interop.layers.bridges import RECORD_EXPRESSION -from bead.interop.layers.graph_lens import graph_to_layers -from bead.interop.layers.model_lenses import ALL_MIRROR_ISOS -from bead.interop.layers.parse_lens import parse_to_layers -from bead.tokenization.parsers import ParsedSentence, ParsedToken - -# Reuse the exact instances exercised by the round-trip suites. -from tests.interop.test_layers_defs import _EXAMPLES as _DEF_EXAMPLES -from tests.interop.test_layers_records import _EXAMPLES as _RECORD_EXAMPLES - -_VALIDATOR = Path(__file__).parent / "lexicon_validator" -_INSTALLED = _VALIDATOR / "node_modules" / "@atproto" / "lexicon" -_REPO_ROOT = Path(__file__).resolve().parents[2] -_LEXICON_DIR = _REPO_ROOT / "vendor" / "layers" / "lexicons" / "pub" / "layers" - -# Mirror model type -> the lexicon URI its JSON must validate against. -_LEX_URI: dict[type, str] = { - m.LayersUuid: "pub.layers.defs#uuid", - m.Feature: "pub.layers.defs#feature", - m.FeatureMap: "pub.layers.defs#featureMap", - m.KnowledgeRef: "pub.layers.defs#knowledgeRef", - m.BoundingBox: "pub.layers.defs#boundingBox", - m.TemporalSpan: "pub.layers.defs#temporalSpan", - m.AgentRef: "pub.layers.defs#agentRef", - m.ObjectRef: "pub.layers.defs#objectRef", - m.LayersSpan: "pub.layers.defs#span", - m.TokenRef: "pub.layers.defs#tokenRef", - m.TokenRefSequence: "pub.layers.defs#tokenRefSequence", - m.Keyframe: "pub.layers.defs#keyframe", - m.SpatioTemporalAnchor: "pub.layers.defs#spatioTemporalAnchor", - m.TemporalEntity: "pub.layers.defs#temporalEntity", - m.TemporalModifier: "pub.layers.defs#temporalModifier", - m.TemporalExpression: "pub.layers.defs#temporalExpression", - m.SpatialEntity: "pub.layers.defs#spatialEntity", - m.SpatialModifier: "pub.layers.defs#spatialModifier", - m.SpatialExpression: "pub.layers.defs#spatialExpression", - m.PageAnchor: "pub.layers.defs#pageAnchor", - m.TextQuoteSelector: "pub.layers.defs#textQuoteSelector", - m.TextPositionSelector: "pub.layers.defs#textPositionSelector", - m.FragmentSelector: "pub.layers.defs#fragmentSelector", - m.ExternalTarget: "pub.layers.defs#externalTarget", - m.Anchor: "pub.layers.defs#anchor", - m.AlignmentLink: "pub.layers.defs#alignmentLink", - m.AnnotationMetadata: "pub.layers.defs#annotationMetadata", - m.LayersConstraint: "pub.layers.defs#constraint", - r.Expression: "pub.layers.expression.expression", - r.Token: "pub.layers.segmentation.defs#token", - r.Tokenization: "pub.layers.segmentation.defs#tokenization", - r.ArgumentRef: "pub.layers.annotation.defs#argumentRef", - r.Annotation: "pub.layers.annotation.defs#annotation", - r.Cluster: "pub.layers.annotation.defs#cluster", - r.AnnotationLayer: "pub.layers.annotation.annotationLayer", - r.GraphNode: "pub.layers.graph.graphNode", - r.GraphEdge: "pub.layers.graph.graphEdge", - r.GraphEdgeEntry: "pub.layers.graph.defs#graphEdgeEntry", - r.GraphEdgeSet: "pub.layers.graph.graphEdgeSet", - r.AudioInfo: "pub.layers.media.defs#audioInfo", - r.VideoInfo: "pub.layers.media.defs#videoInfo", - r.DocumentInfo: "pub.layers.media.defs#documentInfo", - r.RoleSlot: "pub.layers.ontology.defs#roleSlot", - r.TypeDef: "pub.layers.ontology.typeDef", -} - - -@pytest.fixture(scope="module") -def validate_layers(): # noqa: ANN202 - returns an internal validator callable - """Provide a callable validating ``(lexUri, value)`` pairs via @atproto/lexicon.""" - if not _LEXICON_DIR.is_dir(): - pytest.skip( - "layers lexicons missing; run `git submodule update --init vendor/layers`" - ) - node = shutil.which("node") - if node is None: - pytest.skip("node is required for ATProto lexicon validation") - if not _INSTALLED.exists(): - npm = shutil.which("npm") - if npm is None: - pytest.skip("npm is required to install @atproto/lexicon") - proc = subprocess.run( - [npm, "install", "--no-audit", "--no-fund"], - cwd=_VALIDATOR, - capture_output=True, - text=True, - timeout=300, - ) - if proc.returncode != 0 or not _INSTALLED.exists(): - pytest.skip(f"could not install @atproto/lexicon: {proc.stderr[:200]}") - - def _validate(pairs: list[tuple[str, object]]) -> list[dict[str, object]]: - payload = json.dumps([{"lexUri": uri, "value": value} for uri, value in pairs]) - proc = subprocess.run( - [node, str(_VALIDATOR / "validate.mjs")], - input=payload, - capture_output=True, - text=True, - timeout=120, - ) - assert proc.returncode == 0, proc.stderr - return json.loads(proc.stdout) - - return _validate - - -def _failures(results, pairs): # noqa: ANN001, ANN202 - return [ - {"lexUri": uri, "error": res.get("error")} - for (uri, _value), res in zip(pairs, results, strict=True) - if not res["ok"] - ] - - -def test_all_mirror_models_validate(validate_layers) -> None: # noqa: ANN001 - pairs: list[tuple[str, object]] = [] - for example in (*_DEF_EXAMPLES, *_RECORD_EXAMPLES): - lex_uri = _LEX_URI.get(type(example)) - if lex_uri is None: # the Selector union has no standalone lexicon def - continue - pairs.append((lex_uri, ALL_MIRROR_ISOS[type(example)].forward(example))) - assert not _failures(validate_layers(pairs), pairs) - - -def _graph() -> CorpusGraph: - return CorpusGraph( - nodes=( - CorpusNode( - node_id="sub", record=CorpusRecord(text="submission", source_name="r") - ), - CorpusNode(node_id="alice", node_type="entity", label="Alice"), - ), - edges=( - CorpusEdge( - source_id="sub", - target_id="alice", - edge_type="authored-by", - confidence=0.9, - ), - ), - ) - - -def test_graph_bridge_outputs_validate(validate_layers) -> None: # noqa: ANN001 - view = graph_to_layers(_graph()) - assert isinstance(view, dict) - expressions = view["expressions"] - graph_nodes = view["graphNodes"] - assert isinstance(expressions, dict) and isinstance(graph_nodes, dict) - pairs: list[tuple[str, object]] = [] - for expression in expressions.values(): - pairs.append(("pub.layers.expression.expression", expression)) - for graph_node in graph_nodes.values(): - pairs.append(("pub.layers.graph.graphNode", graph_node)) - pairs.append(("pub.layers.graph.graphEdgeSet", view["graphEdgeSet"])) - assert not _failures(validate_layers(pairs), pairs) - - -def test_record_bridge_output_validates(validate_layers) -> None: # noqa: ANN001 - view, _complement = RECORD_EXPRESSION.forward( - CorpusRecord(text="hello", source_name="s", provenance={"author": "a"}) - ) - pairs = [("pub.layers.expression.expression", view)] - assert not _failures(validate_layers(pairs), pairs) - - -def test_parse_bridge_content_validates(validate_layers) -> None: # noqa: ANN001 - sentence = ParsedSentence( - original_text="dogs bark", - tokens=( - ParsedToken( - index=0, - text="dogs", - upos="NOUN", - deprel="nsubj", - head=1, - start_char=0, - end_char=4, - ), - ParsedToken( - index=1, - text="bark", - upos="VERB", - deprel="root", - head=None, - start_char=5, - end_char=9, - ), - ), - ) - view = parse_to_layers(sentence) - assert isinstance(view, dict) - tokenization = view["tokenization"] - pos_layer = view["posLayer"] - dep_layer = view["dependencyLayer"] - assert isinstance(tokenization, dict) - assert isinstance(pos_layer, dict) and isinstance(dep_layer, dict) - pairs: list[tuple[str, object]] = [ - ("pub.layers.segmentation.defs#tokenization", tokenization) - ] - tokens = tokenization["tokens"] - assert isinstance(tokens, tuple) - for token in tokens: - pairs.append(("pub.layers.segmentation.defs#token", token)) - for layer in (pos_layer, dep_layer): - annotations = layer["annotations"] - assert isinstance(annotations, tuple) - for annotation in annotations: - pairs.append(("pub.layers.annotation.defs#annotation", annotation)) - assert not _failures(validate_layers(pairs), pairs) diff --git a/tests/interop/test_layers_parse_iso.py b/tests/interop/test_layers_parse_iso.py index 2d43cb5..3392e34 100644 --- a/tests/interop/test_layers_parse_iso.py +++ b/tests/interop/test_layers_parse_iso.py @@ -4,7 +4,9 @@ from hypothesis import given from hypothesis import strategies as st +from lairs.records import defs +from bead.interop.layers._convert import read_feature_map from bead.interop.layers.parse_lens import PARSED_SENTENCE_LAYERS, parse_to_layers from bead.tokenization.parsers import ParsedSentence, ParsedToken @@ -92,26 +94,31 @@ def test_full_parse(self) -> None: def test_root_head_minus_one(self) -> None: view = parse_to_layers(_known_sentence()) # the root token (index 2) is encoded with headIndex -1 - dep = view["dependencyLayer"]["annotations"][2] - assert dep["headIndex"] == -1 - assert dep["label"] == "root" + dep = view.dependency_layer.annotations[2] + assert dep.headIndex == -1 + assert dep.label == "root" def test_view_is_layers_shaped(self) -> None: view = parse_to_layers(_known_sentence()) - assert set(view) == { - "originalText", - "tokenization", - "posLayer", - "dependencyLayer", - } - assert view["posLayer"]["subkind"] == "pos" - assert view["dependencyLayer"]["subkind"] == "dependency" - assert view["tokenization"]["tokens"][0]["textSpan"] == { - "byteStart": 0, - "byteEnd": 3, - "charStart": 0, - "charEnd": 3, - } + assert view.pos_layer.subkind == "pos" + assert view.dependency_layer.subkind == "dependency" + assert view.tokenization.tokens[0].textSpan == defs.Span( + byteStart=0, byteEnd=3, charStart=0, charEnd=3 + ) + + def test_space_after_rides_in_pos_features(self) -> None: + sentence = ParsedSentence( + original_text="hi", + tokens=( + ParsedToken( + index=0, text="hi", start_char=0, end_char=2, space_after=False + ), + ), + ) + view = parse_to_layers(sentence) + # layers ``token`` has no spaceAfter slot, so it travels in pos features. + features = read_feature_map(view.pos_layer.annotations[0].features) + assert features["spaceAfter"] is False def test_missing_optionals(self) -> None: _assert_roundtrip( diff --git a/tests/interop/test_layers_record_bridge.py b/tests/interop/test_layers_record_bridge.py index dea403b..082f645 100644 --- a/tests/interop/test_layers_record_bridge.py +++ b/tests/interop/test_layers_record_bridge.py @@ -38,9 +38,11 @@ def test_view_is_layers_expression(self) -> None: view = record_to_expression( CorpusRecord(text="hi", source_name="s", provenance={"k": "v"}) ) - assert view["kind"] == "expression" - assert view["text"] == "hi" - assert view["features"]["entries"][0] == {"key": "k", "value": '"v"'} + assert view.kind == "sentence" + assert view.text == "hi" + assert view.features is not None + assert view.features.entries[0].key == "k" + assert view.features.entries[0].value == '"v"' _scalar = st.one_of(st.text(max_size=6), st.integers(-50, 50), st.booleans(), st.none()) diff --git a/tests/interop/test_layers_records.py b/tests/interop/test_layers_records.py deleted file mode 100644 index bc55da9..0000000 --- a/tests/interop/test_layers_records.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Round-trip law tests for the layers record mirror isos.""" - -from __future__ import annotations - -import didactic.api as dx -import pytest - -from bead.interop.layers import models as m -from bead.interop.layers import models_records as r -from bead.interop.layers.model_lenses import RECORD_ISOS - -_UUID = m.LayersUuid(value="u1") -_KR = m.KnowledgeRef(source="wikidata", identifier="Q5") -_REF = m.ObjectRef(local_id=_UUID) -_META = m.AnnotationMetadata(tool="spacy", timestamp="2026-05-29T00:00:00+00:00") -_NOW = "2026-05-29T00:00:00+00:00" - -# One representative instance per record mirror model. -_EXAMPLES: tuple[dx.Model, ...] = ( - r.Expression( - id="doc1", - kind="document", - created_at=_NOW, - text="Hello world.", - anchor=m.Anchor(text_span=m.LayersSpan(byte_start=0, byte_end=12)), - metadata=_META, - features=m.FeatureMap(entries=(m.Feature(key="lang", value="en"),)), - knowledge_refs=(_KR,), - languages=("en",), - ), - r.Token( - token_index=0, text="Hello", text_span=m.LayersSpan(byte_start=0, byte_end=5) - ), - r.Tokenization( - uuid=_UUID, - kind="penn-treebank", - tokens=(r.Token(token_index=0, text="Hi"),), - metadata=_META, - ), - r.ArgumentRef(role="ARG0", target=_REF), - r.Annotation( - uuid=_UUID, - token_index=2, - label="nsubj", - head_index=3, - arguments=(r.ArgumentRef(role="ARG0", target=_REF),), - confidence=900, - knowledge_refs=(_KR,), - temporal=m.TemporalExpression(type="date"), - ), - r.Cluster(uuid=_UUID, canonical_label="Alice", members=(_REF,)), - r.AnnotationLayer( - expression="at://did:plc:abc123/pub.layers.expression.expression/self", - kind="relation", - subkind="dependency", - formalism="universal-dependencies", - created_at=_NOW, - tokenization_id=_UUID, - annotations=(r.Annotation(uuid=_UUID, token_index=0, label="root"),), - metadata=_META, - ), - r.GraphNode( - node_type="entity", - created_at=_NOW, - label="Alice", - properties=m.FeatureMap(entries=(m.Feature(key="k", value="v"),)), - knowledge_refs=(_KR,), - ), - r.GraphEdge( - source=_REF, - target=_REF, - edge_type="coreference", - created_at=_NOW, - ordinal=1, - confidence=800, - ), - r.GraphEdgeEntry(uuid=_UUID, edge_type="reply-to", source=_REF, target=_REF), - r.GraphEdgeSet( - created_at=_NOW, - edges=( - r.GraphEdgeEntry( - uuid=_UUID, edge_type="reply-to", source=_REF, target=_REF - ), - ), - expression="at://did:plc:abc123/pub.layers.expression.expression/self", - ), - r.AudioInfo(sample_rate=44100, channels=2, codec="pcm"), - r.VideoInfo(width=1920, height=1080, frame_rate=30, codec="h264"), - r.DocumentInfo(dpi=300, page_count=12, writing_direction="ltr"), - r.RoleSlot( - role_name="Agent", - filler_type_refs=("at://did:plc:abc123/pub.layers.ontology.typeDef/agent",), - required=True, - constraints=(m.LayersConstraint(expression="x>0"),), - ), - r.TypeDef( - ontology_ref="at://did:plc:abc123/pub.layers.ontology.ontology/self", - name="give", - created_at=_NOW, - type_kind="frame", - allowed_roles=(r.RoleSlot(role_name="Agent"),), - allowed_values=("a", "b"), - ), -) - - -@pytest.mark.parametrize("example", _EXAMPLES, ids=lambda e: type(e).__name__) -def test_record_roundtrip(example: dx.Model) -> None: - iso = RECORD_ISOS[type(example)] - view = iso.forward(example) - assert iso.backward(view) == example - assert iso.forward(iso.backward(view)) == view - - -def test_every_record_has_a_law_passing_iso() -> None: - example_types = {type(example) for example in _EXAMPLES} - assert example_types == set(RECORD_ISOS) - - -def test_annotation_layer_is_camelcased() -> None: - iso = RECORD_ISOS[r.AnnotationLayer] - expression_uri = "at://did:plc:abc123/pub.layers.expression.expression/self" - view = iso.forward( - r.AnnotationLayer( - expression=expression_uri, - kind="relation", - subkind="dependency", - created_at=_NOW, - ) - ) - assert isinstance(view, dict) - assert view["expression"] == expression_uri - assert view["subkind"] == "dependency" - assert "createdAt" in view diff --git a/tests/interop/test_layers_resource.py b/tests/interop/test_layers_resource.py index 0b24f18..28a3916 100644 --- a/tests/interop/test_layers_resource.py +++ b/tests/interop/test_layers_resource.py @@ -25,14 +25,14 @@ def test_full(self) -> None: source="UniMorph", ) view, complement = LEXICAL_ITEM_ENTRY.forward(item) - assert view["form"] == "ran" - assert view["lemma"] == "run" + assert view.form == "ran" + assert view.lemma == "run" assert LEXICAL_ITEM_ENTRY.backward(view, complement) == item def test_form_defaults_to_lemma_in_view(self) -> None: item = LexicalItem(lemma="dog", language_code="eng") view, complement = LEXICAL_ITEM_ENTRY.forward(item) - assert view["form"] == "dog" # faithful entry.form + assert view.form == "dog" # faithful entry.form # but the original None form is recovered exactly restored = LEXICAL_ITEM_ENTRY.backward(view, complement) assert restored.form is None @@ -54,8 +54,8 @@ def test_roundtrip(self) -> None: tags=("motion", "manner"), ) view, complement = LEXICON_COLLECTION.forward(lexicon) - assert view["collection"]["kind"] == "lexicon" - assert len(view["entries"]) == 2 + assert view.collection.kind == "lexicon" + assert len(view.entries) == 2 assert LEXICON_COLLECTION.backward(view, complement) == lexicon def test_empty(self) -> None: @@ -94,11 +94,11 @@ def test_roundtrip(self) -> None: metadata={"source": "manual"}, ) view, complement = TEMPLATE_LAYERS.forward(template) - assert view["text"] == "The {subj} {verb} the {obj}." - assert set(view["slots"]) == {"subj", "verb", "obj"} - assert view["slots"]["verb"]["constraints"][0]["expression"] == ( - "self.pos == 'VERB'" - ) + assert view.text == "The {subj} {verb} the {obj}." + assert {slot.name for slot in view.slots} == {"subj", "verb", "obj"} + verb_slot = next(slot for slot in view.slots if slot.name == "verb") + assert verb_slot.constraints is not None + assert verb_slot.constraints[0].expression == "self.pos == 'VERB'" assert TEMPLATE_LAYERS.backward(view, complement) == template def test_minimal(self) -> None: diff --git a/vendor/layers b/vendor/layers deleted file mode 160000 index 6f3bfef..0000000 --- a/vendor/layers +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6f3bfef92ea69a065a7331e76ca51d90cd8faf62 From 76321e7cb45bfd75621dd82310e2b6f76a17287b Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 20:44:51 -0400 Subject: [PATCH 3/7] Add lairs codec, item bridge, corpus IO, and layers CLI Register a 'bead' lairs codec (lairs.codecs entry point) that round-trips an ItemCollection through the layers schema; map an Item's standoff spans and relations to span/relation AnnotationLayer records via ITEM_LAYERS; ingest a lairs Corpus into bead models and emit bead data as a corpus (materialize to Arrow/Parquet, commit to a repository, publish to a PDS). Add a 'bead layers' CLI group and a docker-PDS integration test for the publish path (gated behind --run-integration). Make the spaCy tokenizer test skip on any import failure (spaCy's pydantic.v1 raises ConfigError, not ImportError, on Python 3.14). --- bead/cli/layers.py | 100 +++++ bead/cli/main.py | 1 + bead/interop/layers/__init__.py | 35 ++ bead/interop/layers/codec.py | 148 +++++++ bead/interop/layers/corpus_io.py | 371 ++++++++++++++++++ bead/interop/layers/item_bridge.py | 336 ++++++++++++++++ pyproject.toml | 3 + tests/conftest.py | 177 ++++++++- tests/interop/test_layers_codec.py | 71 ++++ tests/interop/test_layers_codec_entrypoint.py | 16 + tests/interop/test_layers_corpus_io.py | 111 ++++++ tests/interop/test_layers_item_bridge.py | 170 ++++++++ .../test_layers_publish_integration.py | 78 ++++ tests/items/test_span_labeling.py | 9 +- tests/pds/docker-compose.yml | 36 ++ 15 files changed, 1660 insertions(+), 2 deletions(-) create mode 100644 bead/cli/layers.py create mode 100644 bead/interop/layers/codec.py create mode 100644 bead/interop/layers/corpus_io.py create mode 100644 bead/interop/layers/item_bridge.py create mode 100644 tests/interop/test_layers_codec.py create mode 100644 tests/interop/test_layers_codec_entrypoint.py create mode 100644 tests/interop/test_layers_corpus_io.py create mode 100644 tests/interop/test_layers_item_bridge.py create mode 100644 tests/interop/test_layers_publish_integration.py create mode 100644 tests/pds/docker-compose.yml diff --git a/bead/cli/layers.py b/bead/cli/layers.py new file mode 100644 index 0000000..46fcc56 --- /dev/null +++ b/bead/cli/layers.py @@ -0,0 +1,100 @@ +"""Layers interop commands for the bead CLI. + +Convert a bead ``ItemCollection`` to and from the ``layers`` schema through the +``bead`` lairs codec, materialize a collection as an Arrow/Parquet corpus, and +publish a committed corpus revision to an ATProto PDS. +""" + +from __future__ import annotations + +from pathlib import Path + +import click +from lairs.integrations.codecs import CorpusFragment +from lairs.store import Repository +from rich.console import Console + +from bead.cli.utils import print_info, print_success +from bead.interop.layers.codec import BeadCodec +from bead.interop.layers.corpus_io import ( + items_to_corpus, + materialize_corpus, + publish_corpus, +) +from bead.items.item import ItemCollection + +console = Console() + + +@click.group() +def layers() -> None: + r"""Layers interop commands. + + Convert a bead item collection to and from the ``layers`` schema and + materialize it as an Arrow/Parquet corpus. + + \b + Examples: + $ bead layers encode items.json --out fragment.json + $ bead layers decode fragment.json --out items.json + $ bead layers materialize items.json --out corpus/ + """ + + +@layers.command() +@click.argument("items_file", type=click.Path(exists=True, path_type=Path)) +@click.option("--out", "out_file", type=click.Path(path_type=Path), default=None) +def encode(items_file: Path, out_file: Path | None) -> None: + """Encode an ItemCollection JSON document into a layers fragment.""" + fragment = BeadCodec().decode(items_file.read_text()) + payload = fragment.model_dump_json() + if out_file is None: + console.print_json(payload) + else: + out_file.write_text(payload) + print_success(f"Wrote layers fragment to {out_file}") + + +@layers.command() +@click.argument("fragment_file", type=click.Path(exists=True, path_type=Path)) +@click.option("--out", "out_file", type=click.Path(path_type=Path), default=None) +def decode(fragment_file: Path, out_file: Path | None) -> None: + """Decode a layers fragment back into an ItemCollection JSON document.""" + fragment = CorpusFragment.model_validate_json(fragment_file.read_text()) + payload = BeadCodec().encode(fragment.records) + if out_file is None: + console.print_json(payload) + else: + out_file.write_text(payload) + print_success(f"Wrote item collection to {out_file}") + + +@layers.command() +@click.argument("items_file", type=click.Path(exists=True, path_type=Path)) +@click.option("--out", "out_dir", type=click.Path(path_type=Path), required=True) +@click.option("--name", default="corpus", help="Corpus name") +def materialize(items_file: Path, out_dir: Path, name: str) -> None: + """Materialize an ItemCollection as an Arrow/Parquet layers corpus.""" + collection = ItemCollection.model_validate_json(items_file.read_text()) + corpus = items_to_corpus(collection, corpus_name=name) + out_dir.mkdir(parents=True, exist_ok=True) + paths = materialize_corpus(corpus, out_dir) + for path in paths: + print_info(f" {path}") + print_success(f"Materialized {len(paths)} view(s) to {out_dir}") + + +@layers.command() +@click.argument("repo_path", type=click.Path(exists=True, path_type=Path)) +@click.argument("revision") +@click.option("--to", "to_did", required=True, help="Target PDS DID") +@click.option("--dry-run/--no-dry-run", default=True, help="Plan only (default)") +def publish(repo_path: Path, revision: str, to_did: str, dry_run: bool) -> None: + """Publish a committed corpus revision to a PDS (dry run by default).""" + repo = Repository.open(repo_path) + plan = publish_corpus(repo, revision, to=to_did, dry_run=dry_run) + if dry_run: + print_info(str(plan)) + print_success("Computed publish plan (dry run; nothing written)") + else: + print_success(f"Published revision {revision} to {to_did}") diff --git a/bead/cli/main.py b/bead/cli/main.py index 3268159..5b3a6d0 100644 --- a/bead/cli/main.py +++ b/bead/cli/main.py @@ -430,6 +430,7 @@ def _lazy_load(self, cmd_name: str) -> click.Command: "config": ("bead.cli.config", "config"), "deployment": ("bead.cli.deployment", "deployment"), "items": ("bead.cli.items", "items"), + "layers": ("bead.cli.layers", "layers"), "lists": ("bead.cli.lists", "lists"), "models": ("bead.cli.models", "models"), "protocol": ("bead.cli.protocol", "protocol"), diff --git a/bead/interop/layers/__init__.py b/bead/interop/layers/__init__.py index 69571a4..91d7e2a 100644 --- a/bead/interop/layers/__init__.py +++ b/bead/interop/layers/__init__.py @@ -16,6 +16,9 @@ - a dependency parse to a layers ``tokenization`` with part-of-speech and dependency annotation layers bundled as a :class:`ParsedSentenceLayers` view (see ``parse_lens``). +- an ``Item``'s spans and relations to span and relation annotation layers (see + ``item_bridge``), with a ``bead`` lairs codec over an ``ItemCollection`` (see + ``codec``) and corpus ingest/egress helpers (see ``corpus_io``). - bead's resources to their layers counterparts: ``LexicalItem`` to an ``entry``, ``Lexicon`` to a ``collection``, and ``Template`` to a ``template`` (see ``resource_lens``). @@ -28,12 +31,30 @@ RecordExpressionLens, record_to_expression, ) +from bead.interop.layers.codec import BeadCodec +from bead.interop.layers.corpus_io import ( + corpus_to_graph, + corpus_to_items, + corpus_to_records, + expression_to_record, + graph_to_corpus, + items_to_corpus, + load_layers_corpus, + materialize_corpus, + publish_corpus, + save_corpus_repo, +) from bead.interop.layers.graph_lens import ( CORPUS_GRAPH_LAYERS, CorpusGraphLayers, CorpusGraphLayersLens, graph_to_layers, ) +from bead.interop.layers.item_bridge import ( + ITEM_LAYERS, + ItemLayersLens, + item_to_layers, +) from bead.interop.layers.parse_lens import ( PARSED_SENTENCE_LAYERS, ParsedSentenceLayers, @@ -52,13 +73,16 @@ __all__ = [ "CORPUS_GRAPH_LAYERS", + "ITEM_LAYERS", "LEXICAL_ITEM_ENTRY", "LEXICON_COLLECTION", "PARSED_SENTENCE_LAYERS", "RECORD_EXPRESSION", "TEMPLATE_LAYERS", + "BeadCodec", "CorpusGraphLayers", "CorpusGraphLayersLens", + "ItemLayersLens", "LexicalItemEntryLens", "LexiconCollectionLens", "LexiconLayers", @@ -66,7 +90,18 @@ "ParsedSentenceLayersIso", "RecordExpressionLens", "TemplateLayersLens", + "corpus_to_graph", + "corpus_to_items", + "corpus_to_records", + "expression_to_record", + "graph_to_corpus", "graph_to_layers", + "item_to_layers", + "items_to_corpus", + "load_layers_corpus", + "materialize_corpus", "parse_to_layers", + "publish_corpus", "record_to_expression", + "save_corpus_repo", ] diff --git a/bead/interop/layers/codec.py b/bead/interop/layers/codec.py new file mode 100644 index 0000000..1e15e99 --- /dev/null +++ b/bead/interop/layers/codec.py @@ -0,0 +1,148 @@ +"""A ``lairs`` codec that round-trips a bead ``ItemCollection``. + +:class:`BeadCodec` binds the :class:`lairs.integrations.ports.Codec` port so a +downstream user can reach it through ``lairs.codec("bead")`` once both packages +are installed (the codec is registered via the ``lairs.codecs`` entry point in +bead's project metadata). + +The codec's external format is a bead ``ItemCollection`` serialized as JSON. +``decode`` runs each item through :data:`~bead.interop.layers.item_bridge.ITEM_LAYERS` +and concatenates the resulting canonical ``layers`` records, threading each item's +lens complement and the collection's own fields as private records under the +``bead.interop.complement`` NSID (a layers consumer ignores unknown NSIDs). +Because the complement rides along, ``encode(decode(x)) == x`` holds losslessly +for every collection, and a single canonical mapping (the lens) drives both +directions. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from uuid import UUID + +from lairs.integrations.codecs import CorpusFragment, FragmentRecord + +from bead.data.base import JsonValue +from bead.interop.layers._convert import ( + apply_identity, + identity_of, + j_int, + j_obj, + j_str, +) +from bead.interop.layers.item_bridge import ITEM_LAYERS +from bead.items.item import Item, ItemCollection + +_COMPLEMENT_NSID = "bead.interop.complement" +_COLLECTION_LOCAL_ID = "collection" + + +class BeadCodec: + """Bidirectional codec ``ItemCollection JSON <-> layers corpus fragment``.""" + + name = "bead" + + def decode( + self, src: str | bytes, *, into: CorpusFragment | None = None + ) -> CorpusFragment: + """Decode a bead ``ItemCollection`` JSON document into a layers fragment.""" + collection = ItemCollection.model_validate_json(_as_text(src)) + records: list[FragmentRecord] = list(into.records) if into is not None else [] + records.append( + FragmentRecord( + local_id=_COLLECTION_LOCAL_ID, + nsid=_COMPLEMENT_NSID, + value_json=json.dumps(_collection_complement(collection)), + ) + ) + for index, item in enumerate(collection.items): + fragment, complement = ITEM_LAYERS.forward(item) + prefix = f"item{index}:" + for record in fragment.records: + records.append( + FragmentRecord( + local_id=f"{prefix}{record.local_id}", + nsid=record.nsid, + value_json=record.value_json, + ) + ) + records.append( + FragmentRecord( + local_id=f"{prefix}complement", + nsid=_COMPLEMENT_NSID, + value_json=json.dumps(complement), + ) + ) + return CorpusFragment(records=tuple(records), source="bead") + + def encode(self, records: Iterable[FragmentRecord]) -> str: + """Encode layers fragment records back into ``ItemCollection`` JSON.""" + collection_complement: JsonValue = {} + item_records: dict[int, list[FragmentRecord]] = {} + item_complements: dict[int, JsonValue] = {} + for record in records: + if record.local_id == _COLLECTION_LOCAL_ID: + collection_complement = json.loads(record.value_json) + continue + index, original = _split_index(record.local_id) + if index is None: + continue + if record.nsid == _COMPLEMENT_NSID and original == "complement": + item_complements[index] = json.loads(record.value_json) + else: + item_records.setdefault(index, []).append( + FragmentRecord( + local_id=original, + nsid=record.nsid, + value_json=record.value_json, + ) + ) + items: list[Item] = [] + for index in sorted(item_records): + fragment = CorpusFragment(records=tuple(item_records[index]), source="bead") + items.append(ITEM_LAYERS.backward(fragment, item_complements[index])) + collection = _rebuild_collection(collection_complement, tuple(items)) + return collection.model_dump_json() + + +def _collection_complement(collection: ItemCollection) -> JsonValue: + return { + "identity": identity_of(collection), + "name": collection.name, + "source_template_collection_id": str(collection.source_template_collection_id), + "source_filled_collection_id": str(collection.source_filled_collection_id), + "construction_stats": dict(collection.construction_stats), + } + + +def _rebuild_collection( + complement: JsonValue, items: tuple[Item, ...] +) -> ItemCollection: + comp = j_obj(complement) + stats = comp["construction_stats"] + collection = ItemCollection( + name=j_str(comp["name"]), + source_template_collection_id=UUID(j_str(comp["source_template_collection_id"])), + source_filled_collection_id=UUID(j_str(comp["source_filled_collection_id"])), + items=items, + construction_stats={ + key: j_int(value) + for key, value in (stats.items() if isinstance(stats, dict) else ()) + }, + ) + return apply_identity(collection, comp["identity"]) + + +def _split_index(local_id: str) -> tuple[int | None, str]: + head, _, rest = local_id.partition(":") + if not head.startswith("item") or not rest: + return None, local_id + try: + return int(head.removeprefix("item")), rest + except ValueError: + return None, local_id + + +def _as_text(src: str | bytes) -> str: + return src.decode("utf-8") if isinstance(src, bytes) else src diff --git a/bead/interop/layers/corpus_io.py b/bead/interop/layers/corpus_io.py new file mode 100644 index 0000000..19273c6 --- /dev/null +++ b/bead/interop/layers/corpus_io.py @@ -0,0 +1,371 @@ +"""Ingest a layers ``Corpus`` into bead models and emit bead data as a corpus. + +Ingest (layers to bead): + +- :func:`expression_to_record` / :func:`corpus_to_records` stream a corpus's + expressions as :class:`~bead.corpus.records.CorpusRecord` instances. +- :func:`corpus_to_graph` builds a :class:`~bead.corpus.graph.CorpusGraph` from a + corpus, deriving a ``parent`` edge from each expression's ``parentRef``. +- :func:`corpus_to_items` reconstructs best-effort :class:`~bead.items.item.Item` + instances from a corpus's span and relation annotation layers. +- :func:`load_layers_corpus` is a thin wrapper over :func:`lairs.load_corpus`. + +Egress (bead to layers): + +- :func:`items_to_corpus` and :func:`graph_to_corpus` build a + :class:`lairs.data.Corpus` from bead data, reusing the canonical lenses. +- :func:`materialize_corpus`, :func:`save_corpus_repo`, and :func:`publish_corpus` + delegate to the corresponding ``lairs`` store and publish entry points; the + network-bound publish path is opt-in and defaults to a dry run. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import didactic.api as dx +from lairs.data import Corpus, load_corpus +from lairs.data.dataset import Dataset +from lairs.records import annotation, expression, segmentation +from lairs.records import corpus as corpus_records + +from bead.corpus.graph import CorpusEdge, CorpusGraph, CorpusNode +from bead.corpus.records import CorpusRecord +from bead.interop.layers._convert import ( + ANNOTATION_LAYER_NSID, + CONFIDENCE_SCALE, + CORPUS_NSID, + EXPRESSION_NSID, + MEMBERSHIP_NSID, + SEGMENTATION_NSID, + feature_map, + read_feature_map_scalar, +) +from bead.interop.layers.item_bridge import ITEM_LAYERS +from bead.items.item import Item, ItemCollection +from bead.items.spans import Span, SpanLabel, SpanRelation, SpanSegment + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + from uuid import UUID + + import httpx + from lairs.atproto.pds import PdsClient + from lairs.store import Repository + +_PARENT_EDGE = "parent" +_DEFAULT_ELEMENT = "text" + + +def _rkey(at_uri: str) -> str: + """Return the trailing identifier of an AT-URI (or the string itself).""" + return at_uri.rsplit("/", 1)[-1] + + +def _each[T: dx.Model](dataset: Dataset[T]) -> Iterator[T]: + """Yield a dataset's records one at a time, flattening its batches.""" + for batch in dataset.iter(): + yield from batch + + +def _source(source_name: str | None, expr: expression.Expression) -> str: + """Resolve a per-expression source name, defaulting to the expression id.""" + return source_name if source_name is not None else (expr.id or "layers") + + +# --- ingest: layers -> bead ------------------------------------------------- + + +def expression_to_record( + expr: expression.Expression, *, source_name: str, record_index: int = 0 +) -> CorpusRecord: + """Build a :class:`CorpusRecord` from a layers ``expression``.""" + return CorpusRecord( + text=expr.text if expr.text is not None else "", + source_name=source_name, + record_index=record_index, + provenance=read_feature_map_scalar(expr.features), + ) + + +def corpus_to_records( + corpus: Corpus, *, source_name: str | None = None +) -> Iterator[CorpusRecord]: + """Stream a corpus's member expressions as corpus records.""" + for index, expr in enumerate(_each(corpus.expressions)): + yield expression_to_record( + expr, source_name=_source(source_name, expr), record_index=index + ) + + +def corpus_to_graph(corpus: Corpus, *, source_name: str | None = None) -> CorpusGraph: + """Build a corpus graph from a corpus, with a ``parent`` edge per ``parentRef``.""" + nodes: list[CorpusNode] = [] + edges: list[CorpusEdge] = [] + for index, expr in enumerate(_each(corpus.expressions)): + record = expression_to_record( + expr, source_name=_source(source_name, expr), record_index=index + ) + nodes.append(CorpusNode(node_id=expr.id, node_type=expr.kind, record=record)) + if expr.parentRef is not None: + edges.append( + CorpusEdge( + source_id=expr.id, + target_id=_rkey(expr.parentRef), + edge_type=_PARENT_EDGE, + ) + ) + return CorpusGraph(nodes=tuple(nodes), edges=tuple(edges)) + + +def _span_from_annotation( + item_annotation: annotation.Annotation, element: str +) -> Span | None: + sequence = ( + item_annotation.anchor.tokenRefSequence + if item_annotation.anchor is not None + else None + ) + if sequence is None: + return None + knowledge = item_annotation.knowledgeRefs or () + label_id = knowledge[0].identifier if knowledge else None + confidence = ( + item_annotation.confidence / CONFIDENCE_SCALE + if item_annotation.confidence is not None + else None + ) + label = ( + SpanLabel( + label=item_annotation.label, label_id=label_id, confidence=confidence + ) + if item_annotation.label is not None + else None + ) + return Span( + span_id=item_annotation.uuid.value, + segments=(SpanSegment(element_name=element, indices=sequence.tokenIndexes),), + head_index=sequence.anchorTokenIndex, + label=label, + ) + + +def _relation_from_annotation( + item_annotation: annotation.Annotation, +) -> SpanRelation | None: + arguments = { + argument.role: argument.target.localId.value + for argument in (item_annotation.arguments or ()) + if argument.target.localId is not None + } + if "source" not in arguments or "target" not in arguments: + return None + label = ( + SpanLabel(label=item_annotation.label) + if item_annotation.label is not None + else None + ) + return SpanRelation( + relation_id=item_annotation.uuid.value, + source_span_id=arguments["source"], + target_span_id=arguments["target"], + label=label, + ) + + +def corpus_to_items(corpus: Corpus, *, item_template_id: UUID) -> Iterator[Item]: + """Reconstruct best-effort items from a corpus's annotation layers. + + Third-party corpora carry no bead complement, so item-construction fields + take their defaults; the rendered text, spans, and relations are recovered + from the expression and its span and relation annotation layers. + """ + for expr_with in _each(corpus.with_annotations()): + element = _DEFAULT_ELEMENT + spans: list[Span] = [] + relations: list[SpanRelation] = [] + for layer in expr_with.annotation_layers: + if layer.kind == "span": + spans.extend( + span + for span in ( + _span_from_annotation(item_annotation, element) + for item_annotation in layer.annotations + ) + if span is not None + ) + elif layer.kind == "relation": + relations.extend( + relation + for relation in ( + _relation_from_annotation(item_annotation) + for item_annotation in layer.annotations + ) + if relation is not None + ) + text = expr_with.expression.text + yield Item( + item_template_id=item_template_id, + rendered_elements={element: text if text is not None else ""}, + spans=tuple(spans), + span_relations=tuple(relations), + ) + + +def load_layers_corpus( + uri: str, + *, + source: str = "pds", + pds_client: PdsClient | None = None, + follow_refs: bool = True, +) -> Corpus: + """Load a layers corpus by AT-URI (a thin wrapper over ``lairs.load_corpus``).""" + return load_corpus( + uri, source=source, pds_client=pds_client, follow_refs=follow_refs + ) + + +# --- egress: bead -> layers ------------------------------------------------- + + +def _expression_uri(authority: str, name: str, key: str) -> str: + return f"at://{authority}/{name}/{key}" + + +def items_to_corpus( + collection: ItemCollection, *, corpus_name: str, authority: str = "local" +) -> Corpus: + """Build a layers corpus from a bead item collection. + + Each item's rendered elements become expressions, its tokenizations become + segmentations, and its spans and relations become annotation layers (reusing + :data:`~bead.interop.layers.item_bridge.ITEM_LAYERS`); a membership links each + expression to the corpus, with one corpus record describing the dataset. + """ + corpus_uri = _expression_uri(authority, CORPUS_NSID, corpus_name) + corpus = Corpus.new(corpus_uri) + expression_index = 0 + for item_index, item in enumerate(collection.items): + fragment, _complement = ITEM_LAYERS.forward(item) + local_to_uri: dict[str, str] = {} + for record in fragment.records: + key = f"item{item_index}-{record.local_id.replace(':', '-')}" + uri = _expression_uri(authority, record.nsid, key) + if record.nsid == EXPRESSION_NSID: + expr = expression.Expression.model_validate_json(record.value_json) + local_to_uri[f"at://local/{expr.id}"] = uri + corpus.add_expression(uri, expr) + corpus.add_membership( + _expression_uri(authority, MEMBERSHIP_NSID, key), + corpus_records.Membership( + corpusRef=corpus_uri, + expressionRef=uri, + createdAt=item.created_at, + ordinal=expression_index, + ), + ) + expression_index += 1 + for record in fragment.records: + key = f"item{item_index}-{record.local_id.replace(':', '-')}" + uri = _expression_uri(authority, record.nsid, key) + if record.nsid == SEGMENTATION_NSID: + seg = segmentation.Segmentation.model_validate_json(record.value_json) + corpus.add_record( + uri, + seg.with_( + expression=local_to_uri.get(seg.expression, seg.expression) + ), + ) + elif record.nsid == ANNOTATION_LAYER_NSID: + layer = annotation.AnnotationLayer.model_validate_json( + record.value_json + ) + corpus.add_annotation_layer( + uri, + layer.with_( + expression=local_to_uri.get(layer.expression, layer.expression) + ), + ) + corpus.add_record( + corpus_uri, + corpus_records.Corpus( + name=corpus_name, + createdAt=collection.created_at, + expressionCount=expression_index, + ), + ) + return corpus + + +def graph_to_corpus( + graph: CorpusGraph, *, corpus_name: str, authority: str = "local" +) -> Corpus: + """Build a layers corpus from a corpus graph's record-bearing nodes.""" + corpus_uri = _expression_uri(authority, CORPUS_NSID, corpus_name) + corpus = Corpus.new(corpus_uri) + ordinal = 0 + for node in graph.nodes: + if node.record is None: + continue + uri = _expression_uri(authority, EXPRESSION_NSID, node.node_id) + corpus.add_expression( + uri, + expression.Expression( + id=node.node_id, + kind=node.node_type, + createdAt=node.record.created_at, + text=node.record.text, + features=feature_map(node.record.provenance), + ), + ) + corpus.add_membership( + _expression_uri(authority, MEMBERSHIP_NSID, node.node_id), + corpus_records.Membership( + corpusRef=corpus_uri, + expressionRef=uri, + createdAt=node.record.created_at, + ordinal=ordinal, + ), + ) + ordinal += 1 + corpus.add_record( + corpus_uri, + corpus_records.Corpus( + name=corpus_name, createdAt=graph.created_at, expressionCount=ordinal + ), + ) + return corpus + + +def materialize_corpus(corpus: Corpus, out_dir: Path) -> list[Path]: + """Materialize a corpus to Arrow/Parquet views (delegates to ``lairs``).""" + return corpus.materialize(out_dir) + + +def save_corpus_repo(corpus: Corpus, path: Path) -> str: + """Commit a corpus to a local lairs repository and return the revision.""" + return corpus.save_to_repo(path) + + +def publish_corpus( + repo: Repository, + revision: str, + *, + to: str, + endpoint: str | None = None, + client: httpx.Client | None = None, + dry_run: bool = True, +) -> object: + """Publish a committed corpus revision to a PDS (opt-in; default dry run). + + Returns the ``lairs`` ``PublishPlan``. ``endpoint`` is the PDS base URL and + ``client`` an authorized ``httpx.Client``; both are required for an actual + write (``dry_run=False``). The network-bound publish entry point is imported + lazily so importing this module never pulls the publish stack. + """ + from lairs.author import publish as publish_module # noqa: PLC0415 + + return publish_module.publish( + repo, revision, to=to, endpoint=endpoint, client=client, dry_run=dry_run + ) diff --git a/bead/interop/layers/item_bridge.py b/bead/interop/layers/item_bridge.py new file mode 100644 index 0000000..6d45b7b --- /dev/null +++ b/bead/interop/layers/item_bridge.py @@ -0,0 +1,336 @@ +"""Lens between a bead ``Item`` and canonical layers annotation records. + +A bead :class:`~bead.items.item.Item` carries standoff span and relation +annotations over one or more rendered elements. This lens projects an item to a +:class:`lairs.integrations.codecs.CorpusFragment` of canonical ``lairs`` records: + +- one :class:`lairs.records.expression.Expression` per rendered element, +- one :class:`lairs.records.segmentation.Segmentation` per tokenized element, +- one span :class:`lairs.records.annotation.AnnotationLayer` per element whose + spans anchor by ``tokenRefSequence`` (``head_index`` maps to + ``anchorTokenIndex``; a Wikidata ``label_id`` maps to a ``knowledgeRef``), +- one relation ``AnnotationLayer`` whose annotations carry ``ArgumentRef`` + source/target ``objectRef`` arguments. + +The layers ``token`` has no space-after slot and an ``Item`` carries many fields +``layers`` cannot express, so the round-trip is a ``dx.Lens``: the view captures +the text, tokenization, and a faithful annotation projection; the complement +carries the bead-only remainder (framework identity, the item-construction +fields, the space-after flags, and the spans and relations verbatim), so the +GetPut/PutGet laws hold for every item. +""" + +from __future__ import annotations + +import json +import re +from uuid import UUID + +import didactic.api as dx +from lairs.integrations.codecs import CorpusFragment, FragmentRecord +from lairs.records import annotation, defs, expression, segmentation + +from bead.data.base import JsonValue +from bead.interop.layers._convert import ( + ANNOTATION_LAYER_NSID, + CONFIDENCE_SCALE, + EXPRESSION_NSID, + SEGMENTATION_NSID, + apply_identity, + dumps_meta, + identity_of, + j_list, + j_obj, + j_str, + loads_meta, +) +from bead.items.item import ( + ConstraintSatisfaction, + Item, + ModelOutput, + UnfilledSlot, +) +from bead.items.spans import Span, SpanRelation + +_WIKIDATA_QID = re.compile(r"Q\d+") + + +def _expression_uri(element: str) -> str: + return f"at://local/{element}" + + +def _tokenization_uuid(element: str) -> str: + return f"{element}:tok" + + +def _token_spans( + tokens: tuple[str, ...], space_after: tuple[bool, ...] +) -> list[defs.Span]: + """Compute UTF-8 byte spans for tokens joined by their space-after flags.""" + spans: list[defs.Span] = [] + cursor = 0 + char_cursor = 0 + for index, token in enumerate(tokens): + byte_start = cursor + char_start = char_cursor + cursor += len(token.encode("utf-8")) + char_cursor += len(token) + spans.append( + defs.Span( + byteStart=byte_start, + byteEnd=cursor, + charStart=char_start, + charEnd=char_cursor, + ) + ) + gap = index < len(space_after) and space_after[index] + if gap: + cursor += 1 + char_cursor += 1 + return spans + + +def _knowledge_refs(label_id: str | None, label: str | None) -> tuple[ + defs.KnowledgeRef, ... +]: + if label_id is None: + return () + source = "wikidata" if _WIKIDATA_QID.fullmatch(label_id) else "custom" + return (defs.KnowledgeRef(identifier=label_id, source=source, label=label),) + + +def _span_annotation(span: Span, element: str) -> annotation.Annotation: + indices = tuple( + index + for segment in span.segments + if segment.element_name == element + for index in segment.indices + ) + label = span.label.label if span.label is not None else None + label_id = span.label.label_id if span.label is not None else None + confidence = ( + round(span.label.confidence * CONFIDENCE_SCALE) + if span.label is not None and span.label.confidence is not None + else None + ) + return annotation.Annotation( + uuid=defs.Uuid(value=span.span_id), + anchor=defs.Anchor( + tokenRefSequence=defs.TokenRefSequence( + tokenIndexes=indices, + anchorTokenIndex=span.head_index, + tokenizationId=defs.Uuid(value=_tokenization_uuid(element)), + ) + ), + label=label, + knowledgeRefs=_knowledge_refs(label_id, label), + confidence=confidence, + ) + + +def _relation_annotation(relation: SpanRelation) -> annotation.Annotation: + label = relation.label.label if relation.label is not None else None + return annotation.Annotation( + uuid=defs.Uuid(value=relation.relation_id), + label=label, + arguments=( + annotation.ArgumentRef( + role="source", + target=defs.ObjectRef(localId=defs.Uuid(value=relation.source_span_id)), + ), + annotation.ArgumentRef( + role="target", + target=defs.ObjectRef(localId=defs.Uuid(value=relation.target_span_id)), + ), + ), + ) + + +class ItemLayersLens(dx.Lens[Item, CorpusFragment, JsonValue]): + """Lossless lens ``Item <-> (layers corpus fragment, bead complement)``.""" + + def forward(self, item: Item) -> tuple[CorpusFragment, JsonValue]: + """Project an item to a layers fragment and bead complement.""" + records: list[FragmentRecord] = [] + spans_by_element: dict[str, list[Span]] = {} + for span in item.spans: + if span.segments: + element = span.segments[0].element_name + spans_by_element.setdefault(element, []).append(span) + + for element, text in item.rendered_elements.items(): + records.append( + _record( + EXPRESSION_NSID, + f"expression:{element}", + expression.Expression( + id=element, + kind="sentence", + createdAt=item.created_at, + text=text, + ), + ) + ) + tokens = item.tokenized_elements.get(element) + if tokens: + spaces = item.token_space_after.get(element, ()) + token_spans = _token_spans(tokens, spaces) + tokenization = segmentation.Tokenization( + uuid=defs.Uuid(value=_tokenization_uuid(element)), + kind="custom", + tokens=tuple( + segmentation.Token( + tokenIndex=index, text=token, textSpan=token_spans[index] + ) + for index, token in enumerate(tokens) + ), + ) + records.append( + _record( + SEGMENTATION_NSID, + f"segmentation:{element}", + segmentation.Segmentation( + createdAt=item.created_at, + expression=_expression_uri(element), + tokenizations=(tokenization,), + ), + ) + ) + element_spans = spans_by_element.get(element) + if element_spans: + records.append( + _record( + ANNOTATION_LAYER_NSID, + f"spans:{element}", + annotation.AnnotationLayer( + annotations=tuple( + _span_annotation(span, element) + for span in element_spans + ), + createdAt=item.created_at, + expression=_expression_uri(element), + kind="span", + tokenizationId=defs.Uuid( + value=_tokenization_uuid(element) + ), + ), + ) + ) + + if item.span_relations: + first_element = next(iter(item.rendered_elements), "text") + records.append( + _record( + ANNOTATION_LAYER_NSID, + "relations", + annotation.AnnotationLayer( + annotations=tuple( + _relation_annotation(relation) + for relation in item.span_relations + ), + createdAt=item.created_at, + expression=_expression_uri(first_element), + kind="relation", + ), + ) + ) + + view = CorpusFragment(records=tuple(records), source="bead") + complement: JsonValue = { + "identity": identity_of(item), + "item_template_id": str(item.item_template_id), + "filled_template_refs": tuple( + str(ref) for ref in item.filled_template_refs + ), + "options": item.options, + "unfilled_slots": tuple( + slot.model_dump_json() for slot in item.unfilled_slots + ), + "model_outputs": tuple( + output.model_dump_json() for output in item.model_outputs + ), + "constraint_satisfaction": tuple( + record.model_dump_json() for record in item.constraint_satisfaction + ), + "item_metadata": dumps_meta(item.item_metadata), + "token_space_after": json.dumps( + {key: list(value) for key, value in item.token_space_after.items()} + ), + "spans": tuple(span.model_dump_json() for span in item.spans), + "span_relations": tuple( + relation.model_dump_json() for relation in item.span_relations + ), + } + return view, complement + + def backward(self, view: CorpusFragment, complement: JsonValue) -> Item: + """Reconstruct an item from its layers fragment and bead complement.""" + comp = j_obj(complement) + rendered: dict[str, str] = {} + tokenized: dict[str, tuple[str, ...]] = {} + for record in view.records: + if record.nsid == EXPRESSION_NSID: + expr = expression.Expression.model_validate_json(record.value_json) + rendered[expr.id] = expr.text if expr.text is not None else "" + elif record.nsid == SEGMENTATION_NSID: + seg = segmentation.Segmentation.model_validate_json(record.value_json) + element = seg.expression.removeprefix("at://local/") + tokenization = seg.tokenizations[0] + tokenized[element] = tuple( + token.text if token.text is not None else "" + for token in tokenization.tokens + ) + + token_space_after = { + key: tuple(bool(flag) for flag in value) + for key, value in json.loads( + j_str(comp["token_space_after"]) + ).items() + } + item = Item( + item_template_id=UUID(j_str(comp["item_template_id"])), + filled_template_refs=tuple( + UUID(j_str(ref)) for ref in j_list(comp["filled_template_refs"]) + ), + rendered_elements=rendered, + options=tuple(j_str(option) for option in j_list(comp["options"])), + unfilled_slots=tuple( + UnfilledSlot.model_validate_json(j_str(slot)) + for slot in j_list(comp["unfilled_slots"]) + ), + model_outputs=tuple( + ModelOutput.model_validate_json(j_str(output)) + for output in j_list(comp["model_outputs"]) + ), + constraint_satisfaction=tuple( + ConstraintSatisfaction.model_validate_json(j_str(record)) + for record in j_list(comp["constraint_satisfaction"]) + ), + item_metadata=loads_meta(comp["item_metadata"]), + spans=tuple( + Span.model_validate_json(j_str(span)) + for span in j_list(comp["spans"]) + ), + span_relations=tuple( + SpanRelation.model_validate_json(j_str(relation)) + for relation in j_list(comp["span_relations"]) + ), + tokenized_elements=tokenized, + token_space_after=token_space_after, + ) + return apply_identity(item, comp["identity"]) + + +def _record(nsid: str, local_id: str, model: dx.Model) -> FragmentRecord: + return FragmentRecord( + local_id=local_id, nsid=nsid, value_json=model.model_dump_json() + ) + + +ITEM_LAYERS = ItemLayersLens() + + +def item_to_layers(item: Item) -> CorpusFragment: + """Return the standalone layers fragment view of an item.""" + view, _complement = ITEM_LAYERS.forward(item) + return view diff --git a/pyproject.toml b/pyproject.toml index 2826c55..4828413 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,9 @@ corpus = [ [project.scripts] bead = "bead.cli.main:cli" +[project.entry-points."lairs.codecs"] +bead = "bead.interop.layers.codec:BeadCodec" + [tool.pytest.ini_options] testpaths = ["tests", "docs/user-guide/cli"] norecursedirs = ["tests/fixtures", ".hypothesis"] diff --git a/tests/conftest.py b/tests/conftest.py index 314cb97..9a0efc4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,30 @@ -"""Root pytest configuration for bead package tests.""" +"""Root pytest configuration for bead package tests. + +Adds a ``--run-integration`` flag and an ``integration`` marker. Integration +tests (which stand up a real PDS in docker) are deselected by default; passing +``--run-integration`` opts in. The ``pds_server`` fixture starts the PDS defined +in ``tests/pds`` and skips cleanly when docker, the image, or the health check +are unavailable. +""" from __future__ import annotations +import os +import secrets +import shutil +import socket +import subprocess +import time from pathlib import Path +from typing import TYPE_CHECKING +import didactic.api as dx +import httpx import pytest +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + def pytest_configure(config: pytest.Config) -> None: """Register custom markers.""" @@ -14,6 +33,34 @@ def pytest_configure(config: pytest.Config) -> None: "slow_model_training: marks tests that train ML models " "(deselect with '-m \"not slow_model_training\"')", ) + config.addinivalue_line( + "markers", + "integration: marks tests that need real IO such as a docker PDS " + "(deselected unless --run-integration is given)", + ) + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Register the ``--run-integration`` command-line flag.""" + parser.addoption( + "--run-integration", + action="store_true", + default=False, + help="run integration tests (real IO, docker, credentials)", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, + items: Iterable[pytest.Item], +) -> None: + """Skip integration tests unless ``--run-integration`` was passed.""" + if config.getoption("--run-integration"): + return + skip = pytest.mark.skip(reason="need --run-integration to run") + for item in items: + if "integration" in item.keywords: + item.add_marker(skip) @pytest.fixture(scope="session") @@ -45,3 +92,131 @@ def sample_data_dir(tmp_path: Path) -> Path: data_dir = tmp_path / "data" data_dir.mkdir() return data_dir + + +# --- PDS integration harness ------------------------------------------------- + +_PDS_COMPOSE = Path(__file__).parent / "pds" / "docker-compose.yml" +_PDS_HEALTH_TIMEOUT_S = 90.0 + + +class PdsServer(dx.Model): + """Connection details for a running local PDS test instance. + + Attributes + ---------- + endpoint : str + The base URL of the running PDS. + did : str + The DID of the throwaway account created for the test session. + handle : str + The account handle. + password : str + The account password. + access_jwt : str + The access token authorising writes to the account's repository. + admin_password : str + The PDS admin password generated for this session. + """ + + endpoint: str + did: str + handle: str + password: str + access_jwt: str + admin_password: str + + +def _pds_port() -> int: + override = os.environ.get("BEAD_PDS_PORT") + if override: + return int(override) + with socket.socket() as probe: + probe.bind(("", 0)) + return int(probe.getsockname()[1]) + + +def _docker_available() -> bool: + if shutil.which("docker") is None: + return False + probe = subprocess.run(["docker", "info"], capture_output=True, check=False) + return probe.returncode == 0 + + +def _compose(args: Iterable[str], env: dict[str, str]) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + ["docker", "compose", "-f", str(_PDS_COMPOSE), *args], + capture_output=True, + check=False, + env=env, + ) + + +def _wait_healthy(endpoint: str, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + response = httpx.get(f"{endpoint}/xrpc/_health", timeout=5.0) + except httpx.HTTPError: + response = None + ok = response is not None and response.status_code == httpx.codes.OK + if ok: + return True + time.sleep(2.0) + return False + + +def create_pds_account(endpoint: str, admin_password: str) -> PdsServer: + """Create a throwaway account on the PDS and return its connection details.""" + token = secrets.token_hex(8) + handle = f"u{token}.test" + password = secrets.token_hex(16) + response = httpx.post( + f"{endpoint}/xrpc/com.atproto.server.createAccount", + json={ + "handle": handle, + "email": f"{token}@example.test", + "password": password, + }, + timeout=30.0, + ) + response.raise_for_status() + body = response.json() + return PdsServer( + endpoint=endpoint, + did=str(body["did"]), + handle=str(body.get("handle", handle)), + password=password, + access_jwt=str(body["accessJwt"]), + admin_password=admin_password, + ) + + +@pytest.fixture(scope="session") +def pds_server() -> Iterator[PdsServer]: + """Yield a running local PDS with a throwaway account, or skip cleanly.""" + if not _docker_available(): + pytest.skip("docker is not available") + port = _pds_port() + endpoint = f"http://localhost:{port}" + admin_password = secrets.token_hex(16) + env = { + **os.environ, + "BEAD_PDS_PORT": str(port), + "PDS_JWT_SECRET": secrets.token_hex(16), + "PDS_ADMIN_PASSWORD": admin_password, + "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX": secrets.token_hex(32), + } + started = _compose(["up", "-d"], env) + if started.returncode != 0: + pytest.skip(f"could not start the pds container: {started.stderr.decode()[:300]}") + try: + if not _wait_healthy(endpoint, _PDS_HEALTH_TIMEOUT_S): + pytest.skip("the pds container did not become healthy in time") + try: + server = create_pds_account(endpoint, admin_password) + except httpx.HTTPError as exc: + pytest.skip(f"could not create a test account: {exc}") + yield server + finally: + _compose(["down", "-v"], env) diff --git a/tests/interop/test_layers_codec.py b/tests/interop/test_layers_codec.py new file mode 100644 index 0000000..a0a52f4 --- /dev/null +++ b/tests/interop/test_layers_codec.py @@ -0,0 +1,71 @@ +"""Tests for the BeadCodec lairs codec.""" + +from __future__ import annotations + +import subprocess +import sys +from uuid import uuid4 + +from lairs.integrations.ports import Codec +from lairs.records import annotation + +from bead.interop.layers.codec import BeadCodec +from bead.items.item import Item, ItemCollection +from bead.items.spans import Span, SpanLabel, SpanSegment + + +def _collection() -> ItemCollection: + return ItemCollection( + name="demo", + source_template_collection_id=uuid4(), + source_filled_collection_id=uuid4(), + items=( + Item( + item_template_id=uuid4(), + rendered_elements={"text": "The cat sat"}, + tokenized_elements={"text": ("The", "cat", "sat")}, + token_space_after={"text": (True, True, False)}, + spans=( + Span( + span_id="s1", + segments=(SpanSegment(element_name="text", indices=(1,)),), + label=SpanLabel(label="ANIMAL", label_id="Q5"), + ), + ), + ), + Item(item_template_id=uuid4(), rendered_elements={"text": "Hello"}), + ), + construction_stats={"built": 2}, + ) + + +def test_codec_name_and_protocol() -> None: + assert BeadCodec.name == "bead" + assert isinstance(BeadCodec(), Codec) + + +def test_encode_decode_is_lossless() -> None: + codec = BeadCodec() + source = _collection().model_dump_json() + fragment = codec.decode(source) + assert codec.encode(fragment.records) == source + + +def test_decoded_records_validate_as_lairs_models() -> None: + fragment = BeadCodec().decode(_collection().model_dump_json()) + layers = [ + annotation.AnnotationLayer.model_validate_json(record.value_json) + for record in fragment.records + if record.nsid == "pub.layers.annotation.annotationLayer" + ] + assert any(layer.kind == "span" for layer in layers) + + +def test_importing_bead_does_not_import_lairs() -> None: + # bead.interop.layers is opt-in; importing the top-level package must not + # pull lairs onto the base import path. + code = "import sys; import bead; print('lairs' in sys.modules)" + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert result.stdout.strip() == "False" diff --git a/tests/interop/test_layers_codec_entrypoint.py b/tests/interop/test_layers_codec_entrypoint.py new file mode 100644 index 0000000..beb2015 --- /dev/null +++ b/tests/interop/test_layers_codec_entrypoint.py @@ -0,0 +1,16 @@ +"""The BeadCodec is discoverable through lairs' codec registry.""" + +from __future__ import annotations + +import lairs +from lairs.integrations import registry + +from bead.interop.layers.codec import BeadCodec + + +def test_codec_resolves_by_name() -> None: + assert lairs.codec("bead") is BeadCodec + + +def test_codec_is_listed_as_available() -> None: + assert "bead" in registry.available("codecs") diff --git a/tests/interop/test_layers_corpus_io.py b/tests/interop/test_layers_corpus_io.py new file mode 100644 index 0000000..f5566ad --- /dev/null +++ b/tests/interop/test_layers_corpus_io.py @@ -0,0 +1,111 @@ +"""Tests for the layers corpus ingest and egress helpers.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +from lairs.store import Repository + +from bead.corpus.graph import CorpusGraph, CorpusNode +from bead.corpus.records import CorpusRecord +from bead.interop.layers import corpus_io +from bead.items.item import Item, ItemCollection +from bead.items.spans import Span, SpanLabel, SpanSegment + + +def _collection() -> ItemCollection: + return ItemCollection( + name="demo", + source_template_collection_id=uuid4(), + source_filled_collection_id=uuid4(), + items=( + Item( + item_template_id=uuid4(), + rendered_elements={"text": "The cat sat"}, + tokenized_elements={"text": ("The", "cat", "sat")}, + token_space_after={"text": (True, True, False)}, + spans=( + Span( + span_id="s1", + segments=(SpanSegment(element_name="text", indices=(1,)),), + label=SpanLabel(label="ANIMAL", label_id="Q5"), + ), + ), + ), + ), + ) + + +class TestEgress: + """Bead data projects to a coherent layers corpus.""" + + def test_items_to_corpus_has_expressions(self) -> None: + corpus = corpus_io.items_to_corpus(_collection(), corpus_name="demo") + records = list(corpus_io.corpus_to_records(corpus)) + assert [record.text for record in records] == ["The cat sat"] + + def test_graph_to_corpus_has_expressions(self) -> None: + graph = CorpusGraph( + nodes=( + CorpusNode( + node_id="a", record=CorpusRecord(text="root", source_name="s") + ), + CorpusNode( + node_id="b", + record=CorpusRecord( + text="child", source_name="s", provenance={"score": 3} + ), + ), + ) + ) + corpus = corpus_io.graph_to_corpus(graph, corpus_name="g") + assert len(list(corpus_io.corpus_to_records(corpus))) == 2 + + def test_materialize_writes_parquet(self, tmp_path: Path) -> None: + corpus = corpus_io.items_to_corpus(_collection(), corpus_name="demo") + paths = corpus_io.materialize_corpus(corpus, tmp_path) + names = {path.name for path in paths} + assert "expressions.parquet" in names + assert "annotations.parquet" in names + + def test_save_to_repo_returns_revision(self, tmp_path: Path) -> None: + corpus = corpus_io.items_to_corpus(_collection(), corpus_name="demo") + revision = corpus_io.save_corpus_repo(corpus, tmp_path / "repo") + assert isinstance(revision, str) + assert revision + + def test_publish_dry_run_plans_without_network(self, tmp_path: Path) -> None: + corpus = corpus_io.items_to_corpus(_collection(), corpus_name="demo") + revision = corpus_io.save_corpus_repo(corpus, tmp_path / "repo") + repo = Repository.open(tmp_path / "repo") + plan = corpus_io.publish_corpus( + repo, revision, to="did:plc:example", dry_run=True + ) + assert type(plan).__name__ == "PublishPlan" + assert plan.creates + + +class TestIngest: + """A layers corpus reconstructs into bead corpus models and items.""" + + def test_corpus_to_items_recovers_spans(self) -> None: + corpus = corpus_io.items_to_corpus(_collection(), corpus_name="demo") + items = list(corpus_io.corpus_to_items(corpus, item_template_id=uuid4())) + assert len(items) == 1 + span = items[0].spans[0] + assert span.label is not None + assert span.label.label == "ANIMAL" + assert span.label.label_id == "Q5" + + def test_corpus_to_graph_derives_parent_edges(self) -> None: + graph = CorpusGraph( + nodes=( + CorpusNode( + node_id="a", record=CorpusRecord(text="root", source_name="s") + ), + ) + ) + corpus = corpus_io.graph_to_corpus(graph, corpus_name="g") + rebuilt = corpus_io.corpus_to_graph(corpus) + assert len(rebuilt.nodes) == 1 diff --git a/tests/interop/test_layers_item_bridge.py b/tests/interop/test_layers_item_bridge.py new file mode 100644 index 0000000..b04f8cb --- /dev/null +++ b/tests/interop/test_layers_item_bridge.py @@ -0,0 +1,170 @@ +"""Round-trip law tests for the Item <-> layers annotation fragment lens.""" + +from __future__ import annotations + +from uuid import uuid4 + +from lairs.records import annotation, expression, segmentation + +from bead.interop.layers.item_bridge import ITEM_LAYERS, item_to_layers +from bead.items.item import Item +from bead.items.spans import Span, SpanLabel, SpanRelation, SpanSegment + +LENS = ITEM_LAYERS + + +def _assert_roundtrip(item: Item) -> None: + view, complement = LENS.forward(item) + assert LENS.backward(view, complement) == item + view2, complement2 = LENS.forward(LENS.backward(view, complement)) + assert (view2, complement2) == (view, complement) + + +def _span_layer(item: Item, element: str) -> annotation.AnnotationLayer: + view = item_to_layers(item) + record = next( + record + for record in view.records + if record.local_id == f"spans:{element}" + ) + return annotation.AnnotationLayer.model_validate_json(record.value_json) + + +def _relation_layer(item: Item) -> annotation.AnnotationLayer: + view = item_to_layers(item) + record = next( + record for record in view.records if record.local_id == "relations" + ) + return annotation.AnnotationLayer.model_validate_json(record.value_json) + + +class TestExampleRoundTrips: + """Deterministic round-trips over representative items.""" + + def test_minimal(self) -> None: + _assert_roundtrip( + Item(item_template_id=uuid4(), rendered_elements={"text": "hi"}) + ) + + def test_single_token_span(self) -> None: + _assert_roundtrip( + Item( + item_template_id=uuid4(), + rendered_elements={"text": "The cat sat"}, + tokenized_elements={"text": ("The", "cat", "sat")}, + token_space_after={"text": (True, True, False)}, + spans=( + Span( + span_id="s1", + segments=(SpanSegment(element_name="text", indices=(1,)),), + label=SpanLabel(label="ANIMAL"), + ), + ), + ) + ) + + def test_multi_element_item_with_metadata(self) -> None: + _assert_roundtrip( + Item( + item_template_id=uuid4(), + rendered_elements={"premise": "It rained", "hypothesis": "It was wet"}, + tokenized_elements={ + "premise": ("It", "rained"), + "hypothesis": ("It", "was", "wet"), + }, + token_space_after={"premise": (True, False)}, + options=("entailment", "neutral"), + item_metadata={"pair_id": 7, "tags": ("nli", "weather")}, + ) + ) + + +class TestViewProjection: + """The faithful layers view carries the canonical anchors and arguments.""" + + def test_discontiguous_span_uses_token_ref_sequence(self) -> None: + item = Item( + item_template_id=uuid4(), + rendered_elements={"text": "The big red cat"}, + tokenized_elements={"text": ("The", "big", "red", "cat")}, + spans=( + Span( + span_id="s1", + segments=(SpanSegment(element_name="text", indices=(0, 3)),), + head_index=3, + label=SpanLabel(label="NP"), + ), + ), + ) + layer = _span_layer(item, "text") + anchor = layer.annotations[0].anchor + assert anchor is not None + assert anchor.tokenRefSequence is not None + assert anchor.tokenRefSequence.tokenIndexes == (0, 3) + assert anchor.tokenRefSequence.anchorTokenIndex == 3 + + def test_wikidata_label_id_becomes_knowledge_ref(self) -> None: + item = Item( + item_template_id=uuid4(), + rendered_elements={"text": "Einstein"}, + tokenized_elements={"text": ("Einstein",)}, + spans=( + Span( + span_id="s1", + segments=(SpanSegment(element_name="text", indices=(0,)),), + label=SpanLabel(label="PERSON", label_id="Q937", confidence=0.5), + ), + ), + ) + layer = _span_layer(item, "text") + refs = layer.annotations[0].knowledgeRefs or () + assert refs[0].source == "wikidata" + assert refs[0].identifier == "Q937" + assert layer.annotations[0].confidence == 500 + + def test_relation_uses_argument_refs(self) -> None: + item = Item( + item_template_id=uuid4(), + rendered_elements={"text": "Alice knows Bob"}, + tokenized_elements={"text": ("Alice", "knows", "Bob")}, + spans=( + Span( + span_id="a", + segments=(SpanSegment(element_name="text", indices=(0,)),), + ), + Span( + span_id="b", + segments=(SpanSegment(element_name="text", indices=(2,)),), + ), + ), + span_relations=( + SpanRelation( + relation_id="r1", + source_span_id="a", + target_span_id="b", + label=SpanLabel(label="knows"), + ), + ), + ) + layer = _relation_layer(item) + arguments = layer.annotations[0].arguments or () + roles = { + argument.role: argument.target.localId.value + for argument in arguments + if argument.target.localId is not None + } + assert roles == {"source": "a", "target": "b"} + + def test_records_validate_as_lairs_models(self) -> None: + item = Item( + item_template_id=uuid4(), + rendered_elements={"text": "hi there"}, + tokenized_elements={"text": ("hi", "there")}, + token_space_after={"text": (True, False)}, + ) + view = item_to_layers(item) + for record in view.records: + if record.local_id.startswith("expression:"): + expression.Expression.model_validate_json(record.value_json) + elif record.local_id.startswith("segmentation:"): + segmentation.Segmentation.model_validate_json(record.value_json) diff --git a/tests/interop/test_layers_publish_integration.py b/tests/interop/test_layers_publish_integration.py new file mode 100644 index 0000000..b5eaabc --- /dev/null +++ b/tests/interop/test_layers_publish_integration.py @@ -0,0 +1,78 @@ +"""Integration test: publish a bead corpus to a real PDS and read it back. + +Gated behind ``--run-integration``; the ``pds_server`` fixture stands up a real +bluesky PDS in docker and skips cleanly when docker is unavailable. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING +from uuid import uuid4 + +import httpx +import pytest +from lairs.atproto.pds import PdsClient +from lairs.author.publish import decode +from lairs.records import expression +from lairs.store import Repository + +from bead.interop.layers import corpus_io +from bead.items.item import Item, ItemCollection + +if TYPE_CHECKING: + from tests.conftest import PdsServer + +_EXPRESSION_COLLECTION = "pub.layers.expression.expression" + + +def _collection() -> ItemCollection: + return ItemCollection( + name="published", + source_template_collection_id=uuid4(), + source_filled_collection_id=uuid4(), + items=( + Item(item_template_id=uuid4(), rendered_elements={"text": "dogs bark"}), + ), + ) + + +@pytest.mark.integration +def test_publish_corpus_to_live_pds(pds_server: PdsServer, tmp_path: Path) -> None: + # Mint the corpus under the target account's DID so the records publish into + # that repository, then commit it to a local repo for publishing. + corpus = corpus_io.items_to_corpus( + _collection(), corpus_name="study-1", authority=pds_server.did + ) + revision = corpus_io.save_corpus_repo(corpus, tmp_path / "repo") + repo = Repository.open(tmp_path / "repo") + + headers = {"Authorization": f"Bearer {pds_server.access_jwt}"} + with httpx.Client(headers=headers) as client: + plan = corpus_io.publish_corpus( + repo, + revision, + to=pds_server.did, + endpoint=pds_server.endpoint, + client=client, + dry_run=False, + ) + # The expression record was created on the PDS. + created = [op.uri for op in plan.creates] + expr_uri = next(uri for uri in created if _EXPRESSION_COLLECTION in uri) + rkey = expr_uri.rsplit("/", 1)[-1] + + reader = PdsClient(pds_server.endpoint, client) + envelope = reader.get_record(pds_server.did, _EXPRESSION_COLLECTION, rkey) + assert decode(envelope, expression.Expression).text == "dogs bark" + + # Re-publishing the unchanged revision is a no-op (idempotent). + again = corpus_io.publish_corpus( + repo, + revision, + to=pds_server.did, + endpoint=pds_server.endpoint, + client=client, + dry_run=True, + ) + assert again.is_empty() diff --git a/tests/items/test_span_labeling.py b/tests/items/test_span_labeling.py index 2f07081..07e33e4 100644 --- a/tests/items/test_span_labeling.py +++ b/tests/items/test_span_labeling.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib from uuid import uuid4 import didactic.api as dx @@ -299,7 +300,13 @@ def test_multiple_elements(self) -> None: def test_default_config(self) -> None: """Test tokenizing with default config.""" - pytest.importorskip("spacy") + # ``pytest.importorskip`` only catches ImportError, but spaCy's bundled + # pydantic.v1 raises a ConfigError on Python 3.14, so skip on any import + # failure rather than letting it surface as a test error. + try: + importlib.import_module("spacy") + except Exception as error: + pytest.skip(f"spaCy is not importable on this platform: {error}") item = Item( item_template_id=uuid4(), rendered_elements={"text": "Hello"}, diff --git a/tests/pds/docker-compose.yml b/tests/pds/docker-compose.yml new file mode 100644 index 0000000..86888cc --- /dev/null +++ b/tests/pds/docker-compose.yml @@ -0,0 +1,36 @@ +# Local Personal Data Server for integration testing. +# +# Brings up a real bluesky PDS so the bead layers publish path can be exercised +# against genuine XRPC and repository endpoints rather than a mock transport. +# The container is managed by the `pds_server` fixture in tests/conftest.py, +# which generates the secrets below at run time and passes them through the +# environment. The fixture skips cleanly when docker, the image, or the health +# check are unavailable, so this never hard-fails a test run. + +services: + pds: + image: ${BEAD_PDS_IMAGE:-ghcr.io/bluesky-social/pds:0.4} + # the listen port, published port, and advertised port are all driven by the + # one BEAD_PDS_PORT variable so the endpoint the PDS publishes in its DID + # document (http://localhost:) always equals the endpoint the host + # reaches it on. + ports: + - "${BEAD_PDS_PORT:-3000}:${BEAD_PDS_PORT:-3000}" + environment: + PDS_HOSTNAME: localhost + PDS_PORT: "${BEAD_PDS_PORT:-3000}" + PDS_DATA_DIRECTORY: /pds + PDS_BLOBSTORE_DISK_LOCATION: /pds/blocks + PDS_DID_PLC_URL: ${PDS_DID_PLC_URL:-https://plc.directory} + PDS_JWT_SECRET: ${PDS_JWT_SECRET:-dev-jwt-secret} + PDS_ADMIN_PASSWORD: ${PDS_ADMIN_PASSWORD:-dev-admin-password} + PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX: ${PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX:-0000000000000000000000000000000000000000000000000000000000000001} + PDS_INVITE_REQUIRED: "false" + PDS_SERVICE_HANDLE_DOMAINS: .test + # dev mode relaxes the OAuth resource-URL https requirement so the server + # starts on a plain http://localhost endpoint for local testing. + PDS_DEV_MODE: "true" + LOG_ENABLED: "false" + # an ephemeral data directory; nothing survives `docker compose down`. + tmpfs: + - /pds From 7de14d20817a1c4f54783648947cc048b82039f4 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 20:45:12 -0400 Subject: [PATCH 4/7] Document layers interop and release 0.7.0 Rewrite the layers interop docs for the lairs-backed mapping, codec, item bridge, corpus IO, and CLI; drop the retired mirror-model and submodule references from the README and add a note on the --run-integration PDS tests. Bump the version to 0.7.0 with a changelog entry. --- CHANGELOG.md | 36 ++++++ README.md | 14 +-- bead/__init__.py | 2 +- docs/api/interop.md | 27 ++-- docs/user-guide/api/layers-interop.md | 169 +++++++++++++++++++------- pyproject.toml | 2 +- uv.lock | 2 +- 7 files changed, 183 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170c5c8..4c663f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.0] - 2026-06-29 + +### Added + +#### Deep `layers` integration through `lairs` + +- `bead.interop.layers` builds its layers projections from the canonical + `lairs.records` models that `lairs` generates from the layers lexicons, so the + schema has one source of truth. The bridge, parse, graph, and resource lenses + return typed `lairs` models (or small bead-side aggregates of them) as their + view; the lens complement carries the bead-only remainder, and the round-trip + is guaranteed by the didactic GetPut/PutGet laws. +- `ItemLayersLens` (`ITEM_LAYERS`, `item_to_layers`) maps an `Item`'s standoff + spans and relations to span and relation `AnnotationLayer` records over + per-element expressions and tokenizations: a span anchors by + `tokenRefSequence` (with `head_index` as `anchorTokenIndex` and a Wikidata + `label_id` as a `knowledgeRef`); a relation carries `ArgumentRef` source and + target arguments. +- `BeadCodec`, registered on the `lairs.codecs` entry point, round-trips a bead + `ItemCollection` to and from the layers schema. `lairs.codec("bead")` resolves + it, and `encode(decode(x)) == x` holds losslessly. +- `bead.interop.layers.corpus_io` ingests a `lairs.data.Corpus` into bead corpus + models (`corpus_to_records`, `corpus_to_graph`, `corpus_to_items`) and emits + bead data as a corpus (`items_to_corpus`, `graph_to_corpus`, + `materialize_corpus`, `save_corpus_repo`, `publish_corpus`). +- `bead layers` CLI commands: `encode`, `decode`, `materialize`, and an opt-in + `publish` that defaults to a dry run. + +### Changed + +- `lairs>=0.5.0` is a dependency; the minimum `didactic` is raised to `>=0.9.0` + and `panproto` to `>=0.56.0`. Importing `bead` does not import `lairs`; the + dependency loads only when `bead.interop.layers` is used. +- Layers record validation is owned by `lairs`, whose generated models validate + on construction. + ## [0.6.0] - 2026-05-29 ### Added diff --git a/README.md b/README.md index 7e1daf1..001da2b 100644 --- a/README.md +++ b/README.md @@ -30,20 +30,20 @@ uv pip install bead[training] # PyTorch Lightning, TensorBoard ### Development ```bash -git clone --recurse-submodules https://github.com/FACTSlab/bead.git +git clone https://github.com/FACTSlab/bead.git cd bead uv sync --all-extras uv run pytest tests/ ``` -The `vendor/layers` submodule holds the layers lexicons that the interop tests -validate against. If you cloned without `--recurse-submodules`, fetch them with -`git submodule update --init vendor/layers`, and refresh to the latest published -lexicons with `git submodule update --remote vendor/layers`. The lexicon -validation tests skip automatically when the submodule is absent. - Always use `uv run` to execute commands. +Integration tests that exercise the layers publish path against a real ATProto +PDS are deselected by default. With docker running, opt in with +`uv run pytest --run-integration`; the test stands up a throwaway +[bluesky PDS](https://github.com/bluesky-social/pds) container and skips cleanly +when docker is unavailable. + ## Quick Start ```python diff --git a/bead/__init__.py b/bead/__init__.py index b9111f0..fce6111 100644 --- a/bead/__init__.py +++ b/bead/__init__.py @@ -6,6 +6,6 @@ from __future__ import annotations -__version__ = "0.6.0" +__version__ = "0.7.0" __author__ = "Aaron Steven White" __email__ = "aaron.white@rochester.edu" diff --git a/docs/api/interop.md b/docs/api/interop.md index 4973760..5671fcf 100644 --- a/docs/api/interop.md +++ b/docs/api/interop.md @@ -1,10 +1,10 @@ # bead.interop -Lossless, law-verified interoperability mappings between bead models and -external schemas. The `layers` subpackage maps bead's corpus and annotation data -to the [layers](https://github.com/layers-pub/layers) linguistic-annotation -schema and back via didactic lenses (`dx.Iso` / `dx.Lens`); round-trip fidelity -is guaranteed by the GetPut/PutGet lens laws. +Lossless, law-verified interoperability mappings between bead models and the +[layers](https://github.com/layers-pub/layers) linguistic-annotation schema. The +`layers` subpackage maps bead's corpus, annotation, and resource data to and from +the canonical `lairs.records` models via didactic lenses (`dx.Iso` / `dx.Lens`); +round-trip fidelity is guaranteed by the GetPut/PutGet lens laws. See the [Layers Interoperability guide](../user-guide/api/layers-interop.md) for usage. @@ -26,23 +26,26 @@ usage. show_root_heading: true show_source: false -## Mirror models +::: bead.interop.layers.item_bridge + options: + show_root_heading: true + show_source: false -Faithful didactic mirrors of the layers constructs. +## Resource lenses -::: bead.interop.layers.models +::: bead.interop.layers.resource_lens options: show_root_heading: true show_source: false -::: bead.interop.layers.models_records +## Codec and corpus I/O + +::: bead.interop.layers.codec options: show_root_heading: true show_source: false -## Generic mirror iso - -::: bead.interop.layers.model_lenses +::: bead.interop.layers.corpus_io options: show_root_heading: true show_source: false diff --git a/docs/user-guide/api/layers-interop.md b/docs/user-guide/api/layers-interop.md index 1be5d25..3ed9cc7 100644 --- a/docs/user-guide/api/layers-interop.md +++ b/docs/user-guide/api/layers-interop.md @@ -1,30 +1,34 @@ # Layers Interoperability -bead maps its corpus and annotation data to the +bead maps its corpus, annotation, and resource data to the [layers](https://github.com/layers-pub/layers) linguistic-annotation schema and back **losslessly**, via didactic lenses (`dx.Iso` / `dx.Lens`). The forward -direction produces faithful, standalone layers-shaped JSON; the reverse -reconstructs the exact bead value. Because the mappings are lenses, the -round-trip is guaranteed by the didactic GetPut/PutGet laws (verified in the -test suite with `verify_iso` / `check_lens_laws`). +direction produces a faithful, standalone projection built from the canonical +`layers` record models that [`lairs`](https://pypi.org/project/lairs/) generates +from the layers lexicons; the reverse reconstructs the exact bead value. Because +the mappings are lenses, the round-trip is guaranteed by the didactic +GetPut/PutGet laws (verified in the test suite). -There is no ATProto wire/network dependency: the lenses produce and consume -plain layers-shaped Python/JSON. +bead depends on `lairs` directly, so the layers record models come from one +canonical source (`lairs.records`) rather than a hand-maintained copy. Importing +`bead` does not import `lairs`; the dependency loads only when you reach into +`bead.interop.layers`. ## What is covered -- Every linguistic `pub.layers` construct is mirrored as a faithful didactic - model in `bead.interop.layers.models` / `models_records` (the anchor union, - temporal/spatial expressions, token/text/page/external anchors, the - polymorphic annotation and annotation layer, the property graph, media - descriptors, ontology type definitions, knowledge references, and the shared - objects). Each has a lossless `MirrorIso` to layers JSON. -- bead's own pipeline outputs bridge directly to layers: - - `CorpusGraph` ↔ a layers property graph (expressions + graph nodes + a - `graphEdgeSet`). - - `CorpusRecord` ↔ a layers `expression`. - - a dependency `ParsedSentence` ↔ a layers `tokenization` plus part-of-speech - and dependency annotation layers. +bead maps its pipeline outputs and resources onto the canonical `lairs.records` +models (`expression`, `segmentation`, `annotation`, `graph`, `resource`, and the +shared `defs` objects): + +- `CorpusRecord` to a layers `expression`. +- `CorpusGraph` to the layers property graph (expressions, graph nodes, and a + `graphEdgeSet`), bundled as a `CorpusGraphLayers` view. +- a dependency `ParsedSentence` to a layers `tokenization` plus part-of-speech + and dependency annotation layers, bundled as a `ParsedSentenceLayers` view. +- an `Item`'s span and relation annotations to span and relation + `AnnotationLayer` records over per-element expressions and tokenizations. +- bead resources to their layers counterparts: `LexicalItem` to an `entry`, + `Lexicon` to a `collection`, and `Template` to a `template`. ## Mapping a corpus graph @@ -51,9 +55,10 @@ graph = assemble_graph( ], ) -# Faithful, standalone layers-shaped projection. +# Faithful, standalone layers projection of canonical lairs models. view = graph_to_layers(graph) -assert set(view) == {"expressions", "graphNodes", "graphEdgeSet"} +assert view.expressions[0].kind == "expression" +assert view.edge_set.edges[0].edgeType == "reply-to" # Lossless round-trip via the lens (view + complement reconstruct exactly). layers_view, complement = CORPUS_GRAPH_LAYERS.forward(graph) @@ -91,45 +96,115 @@ sentence = ParsedSentence( ) view = parse_to_layers(sentence) -assert view["dependencyLayer"]["subkind"] == "dependency" +assert view.dependency_layer.subkind == "dependency" # The root token is encoded with headIndex -1 (the layers convention). -assert view["dependencyLayer"]["annotations"][1]["headIndex"] == -1 +assert view.dependency_layer.annotations[1].headIndex == -1 # Iso: the parse reconstructs exactly (no complement needed). assert PARSED_SENTENCE_LAYERS.backward(view) == sentence ``` -## Working with the mirror models directly +The layers `token` has no space-after slot, so each token's `space_after` flag +travels in its part-of-speech annotation's features. + +## Mapping an item's spans -Any layers construct can be built as a bead model and serialized to/from layers -JSON with its `MirrorIso`: +An `Item`'s standoff spans and relations project to span and relation +`AnnotationLayer` records. A span anchors by `tokenRefSequence` (its +`head_index` becomes the sequence's `anchorTokenIndex`, and a Wikidata +`label_id` becomes a `knowledgeRef`); a relation carries `ArgumentRef` source and +target arguments. ```python -from bead.interop.layers import mirror_iso -from bead.interop.layers.models import Anchor, LayersUuid, TokenRef +from uuid import uuid4 + +from bead.interop.layers import ITEM_LAYERS, item_to_layers +from bead.items.item import Item +from bead.items.spans import Span, SpanLabel, SpanSegment + +item = Item( + item_template_id=uuid4(), + rendered_elements={"text": "Einstein won"}, + tokenized_elements={"text": ("Einstein", "won")}, + spans=( + Span( + span_id="s1", + segments=(SpanSegment(element_name="text", indices=(0,)),), + label=SpanLabel(label="PERSON", label_id="Q937"), + ), + ), +) + +fragment = item_to_layers(item) # a lairs CorpusFragment of canonical records +layers_view, complement = ITEM_LAYERS.forward(item) +assert ITEM_LAYERS.backward(layers_view, complement) == item +``` + +## Using bead as a lairs codec + +bead registers a `bead` codec on the `lairs.codecs` entry point, so any tool with +both packages installed can round-trip a bead `ItemCollection` through the +`layers` schema: + +```python +from uuid import uuid4 + +import lairs + +from bead.items.item import Item, ItemCollection -anchor = Anchor( - token_ref=TokenRef(tokenization_id=LayersUuid(value="tok-1"), token_index=4) +collection = ItemCollection( + name="study-1", + source_template_collection_id=uuid4(), + source_filled_collection_id=uuid4(), + items=(Item(item_template_id=uuid4(), rendered_elements={"text": "dogs bark"}),), ) -iso = mirror_iso(Anchor) -layers_json = iso.forward(anchor) -assert layers_json["tokenRef"]["tokenIndex"] == 4 # camelCased, layers-shaped -assert iso.backward(layers_json) == anchor # exact round-trip +codec = lairs.codec("bead")() +fragment = codec.decode(collection.model_dump_json()) +assert codec.encode(fragment.records) == collection.model_dump_json() # lossless ``` -`bead.interop.layers.ALL_MIRROR_ISOS` maps every mirror model type to its iso, -and a coverage test guards that every targeted layers construct has a -law-passing mapping. +## Loading and emitting a corpus + +`bead.interop.layers.corpus_io` ingests a `lairs.data.Corpus` into bead models +and emits bead data as a corpus: + +```python +from pathlib import Path +from uuid import uuid4 + +from bead.interop.layers import corpus_io +from bead.items.item import Item, ItemCollection + +collection = ItemCollection( + name="study-1", + source_template_collection_id=uuid4(), + source_filled_collection_id=uuid4(), + items=(Item(item_template_id=uuid4(), rendered_elements={"text": "dogs bark"}),), +) + +corpus = corpus_io.items_to_corpus(collection, corpus_name="study-1") +paths = corpus_io.materialize_corpus(corpus, Path("corpus_views")) # Arrow/Parquet +revision = corpus_io.save_corpus_repo(corpus, Path("corpus_repo")) # VCS commit + +graph = corpus_io.corpus_to_graph(corpus) # parentRef -> parent edges +records = list(corpus_io.corpus_to_records(corpus)) +``` + +The same operations are available from the command line: + +```console +$ bead layers encode items.json --out fragment.json +$ bead layers decode fragment.json --out items.json +$ bead layers materialize items.json --out corpus/ +``` -## Validating against the layers lexicons +The PDS publish path (`corpus_io.publish_corpus` and `bead layers publish`) is +opt-in and defaults to a dry run. -The mappings are checked against the canonical layers lexicons, vendored as the -`vendor/layers` git submodule pointing at -[`layers-pub/layers`](https://github.com/layers-pub/layers). The interop test -suite feeds every mapping's output through the ATProto lexicon validator -(`@atproto/lexicon`) and asserts each record validates against its lexicon, so a -schema drift in layers surfaces as a failing test. +## Validation -Fetch the lexicons with `git submodule update --init vendor/layers`, and pull the -latest published schemas with `git submodule update --remote vendor/layers`. The -validation tests skip when the submodule is not checked out. +The lenses construct real `lairs.records` models, which validate their structure, +required fields, and types on construction. Conformance to the layers lexicons is +owned upstream by `lairs`, which generates those models from the lexicons and +tests them in its own suite, so bead does not re-host a separate validator. diff --git a/pyproject.toml b/pyproject.toml index 4828413..aac963d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bead" -version = "0.6.0" +version = "0.7.0" description = "Lexicon and Template Collection Construction Pipeline for Acceptability and Inference Judgment Data" authors = [{name = "Aaron Steven White", email = "aaron.white@rochester.edu"}] readme = "README.md" diff --git a/uv.lock b/uv.lock index fd25d19..93ff938 100644 --- a/uv.lock +++ b/uv.lock @@ -170,7 +170,7 @@ wheels = [ [[package]] name = "bead" -version = "0.6.0" +version = "0.7.0" source = { editable = "." } dependencies = [ { name = "accelerate" }, From 21e0b5d3e1d086639bcbc20e5430ffdb1cd4c62b Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 20:54:23 -0400 Subject: [PATCH 5/7] Apply ruff format to the layers interop modules --- bead/interop/layers/codec.py | 4 +++- bead/interop/layers/corpus_io.py | 4 +--- bead/interop/layers/graph_lens.py | 4 +--- bead/interop/layers/item_bridge.py | 17 ++++++----------- bead/interop/layers/resource_lens.py | 4 +--- tests/conftest.py | 8 ++++++-- tests/interop/test_layers_item_bridge.py | 8 ++------ 7 files changed, 20 insertions(+), 29 deletions(-) diff --git a/bead/interop/layers/codec.py b/bead/interop/layers/codec.py index 1e15e99..a3e0a39 100644 --- a/bead/interop/layers/codec.py +++ b/bead/interop/layers/codec.py @@ -123,7 +123,9 @@ def _rebuild_collection( stats = comp["construction_stats"] collection = ItemCollection( name=j_str(comp["name"]), - source_template_collection_id=UUID(j_str(comp["source_template_collection_id"])), + source_template_collection_id=UUID( + j_str(comp["source_template_collection_id"]) + ), source_filled_collection_id=UUID(j_str(comp["source_filled_collection_id"])), items=items, construction_stats={ diff --git a/bead/interop/layers/corpus_io.py b/bead/interop/layers/corpus_io.py index 19273c6..e13c0f1 100644 --- a/bead/interop/layers/corpus_io.py +++ b/bead/interop/layers/corpus_io.py @@ -137,9 +137,7 @@ def _span_from_annotation( else None ) label = ( - SpanLabel( - label=item_annotation.label, label_id=label_id, confidence=confidence - ) + SpanLabel(label=item_annotation.label, label_id=label_id, confidence=confidence) if item_annotation.label is not None else None ) diff --git a/bead/interop/layers/graph_lens.py b/bead/interop/layers/graph_lens.py index 4d0e115..05fd987 100644 --- a/bead/interop/layers/graph_lens.py +++ b/bead/interop/layers/graph_lens.py @@ -142,9 +142,7 @@ def forward(self, graph_in: CorpusGraph) -> tuple[CorpusGraphLayers, JsonValue]: } return view, complement - def backward( - self, view: CorpusGraphLayers, complement: JsonValue - ) -> CorpusGraph: + def backward(self, view: CorpusGraphLayers, complement: JsonValue) -> CorpusGraph: """Reconstruct the graph from its layers view and bead complement.""" comp = j_obj(complement) node_complements = j_obj(comp["node_complements"]) diff --git a/bead/interop/layers/item_bridge.py b/bead/interop/layers/item_bridge.py index 6d45b7b..6613b3a 100644 --- a/bead/interop/layers/item_bridge.py +++ b/bead/interop/layers/item_bridge.py @@ -90,9 +90,9 @@ def _token_spans( return spans -def _knowledge_refs(label_id: str | None, label: str | None) -> tuple[ - defs.KnowledgeRef, ... -]: +def _knowledge_refs( + label_id: str | None, label: str | None +) -> tuple[defs.KnowledgeRef, ...]: if label_id is None: return () source = "wikidata" if _WIKIDATA_QID.fullmatch(label_id) else "custom" @@ -210,9 +210,7 @@ def forward(self, item: Item) -> tuple[CorpusFragment, JsonValue]: createdAt=item.created_at, expression=_expression_uri(element), kind="span", - tokenizationId=defs.Uuid( - value=_tokenization_uuid(element) - ), + tokenizationId=defs.Uuid(value=_tokenization_uuid(element)), ), ) ) @@ -283,9 +281,7 @@ def backward(self, view: CorpusFragment, complement: JsonValue) -> Item: token_space_after = { key: tuple(bool(flag) for flag in value) - for key, value in json.loads( - j_str(comp["token_space_after"]) - ).items() + for key, value in json.loads(j_str(comp["token_space_after"])).items() } item = Item( item_template_id=UUID(j_str(comp["item_template_id"])), @@ -308,8 +304,7 @@ def backward(self, view: CorpusFragment, complement: JsonValue) -> Item: ), item_metadata=loads_meta(comp["item_metadata"]), spans=tuple( - Span.model_validate_json(j_str(span)) - for span in j_list(comp["spans"]) + Span.model_validate_json(j_str(span)) for span in j_list(comp["spans"]) ), span_relations=tuple( SpanRelation.model_validate_json(j_str(relation)) diff --git a/bead/interop/layers/resource_lens.py b/bead/interop/layers/resource_lens.py index 107e1b5..864c333 100644 --- a/bead/interop/layers/resource_lens.py +++ b/bead/interop/layers/resource_lens.py @@ -237,9 +237,7 @@ def forward(self, template: Template) -> tuple[resource.Template, JsonValue]: } return view, complement - def backward( - self, view: resource.Template, complement: JsonValue - ) -> Template: + def backward(self, view: resource.Template, complement: JsonValue) -> Template: """Reconstruct a template from its layers template and complement.""" comp = j_obj(complement) slot_complements = j_obj(comp["slot_complements"]) diff --git a/tests/conftest.py b/tests/conftest.py index 9a0efc4..6e052ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -143,7 +143,9 @@ def _docker_available() -> bool: return probe.returncode == 0 -def _compose(args: Iterable[str], env: dict[str, str]) -> subprocess.CompletedProcess[bytes]: +def _compose( + args: Iterable[str], env: dict[str, str] +) -> subprocess.CompletedProcess[bytes]: return subprocess.run( ["docker", "compose", "-f", str(_PDS_COMPOSE), *args], capture_output=True, @@ -209,7 +211,9 @@ def pds_server() -> Iterator[PdsServer]: } started = _compose(["up", "-d"], env) if started.returncode != 0: - pytest.skip(f"could not start the pds container: {started.stderr.decode()[:300]}") + pytest.skip( + f"could not start the pds container: {started.stderr.decode()[:300]}" + ) try: if not _wait_healthy(endpoint, _PDS_HEALTH_TIMEOUT_S): pytest.skip("the pds container did not become healthy in time") diff --git a/tests/interop/test_layers_item_bridge.py b/tests/interop/test_layers_item_bridge.py index b04f8cb..31b882c 100644 --- a/tests/interop/test_layers_item_bridge.py +++ b/tests/interop/test_layers_item_bridge.py @@ -23,18 +23,14 @@ def _assert_roundtrip(item: Item) -> None: def _span_layer(item: Item, element: str) -> annotation.AnnotationLayer: view = item_to_layers(item) record = next( - record - for record in view.records - if record.local_id == f"spans:{element}" + record for record in view.records if record.local_id == f"spans:{element}" ) return annotation.AnnotationLayer.model_validate_json(record.value_json) def _relation_layer(item: Item) -> annotation.AnnotationLayer: view = item_to_layers(item) - record = next( - record for record in view.records if record.local_id == "relations" - ) + record = next(record for record in view.records if record.local_id == "relations") return annotation.AnnotationLayer.model_validate_json(record.value_json) From 87f7009af40649e4d3893184b71f8b688d3eec76 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 21:37:55 -0400 Subject: [PATCH 6/7] ci: install lairs from source in the Python jobs lairs 0.5.0 is not yet on PyPI and the pip-based CI jobs ignore the tool.uv.sources path redirect, so check out the public layers-pub/lairs repo and pip-install it before installing bead, satisfying the lairs>=0.5.0 requirement. --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f18f968..ffbf087 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,15 @@ jobs: steps: - uses: actions/checkout@v4 + # lairs 0.5.0 is not yet on PyPI; install it from its public source repo + # (mirrors the tool.uv.sources path used for local development). + - name: Check out lairs + uses: actions/checkout@v4 + with: + repository: layers-pub/lairs + ref: main + path: lairs + - name: Set up Python uses: actions/setup-python@v5 with: @@ -169,6 +178,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip + pip install ./lairs pip install -e ".[dev,behavioral-analysis]" - name: Run pyright @@ -181,6 +191,15 @@ jobs: steps: - uses: actions/checkout@v4 + # lairs 0.5.0 is not yet on PyPI; install it from its public source repo + # (mirrors the tool.uv.sources path used for local development). + - name: Check out lairs + uses: actions/checkout@v4 + with: + repository: layers-pub/lairs + ref: main + path: lairs + - name: Install pnpm uses: pnpm/action-setup@v4 with: @@ -223,6 +242,7 @@ jobs: - name: Install dependencies in venv run: | .venv/bin/pip install --upgrade pip + .venv/bin/pip install ./lairs .venv/bin/pip install -e ".[dev,api,training,stats,behavioral-analysis]" - name: Set up test fixtures From 4886e2b8e90ba8d4d606314166863e259a5e1c51 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Mon, 29 Jun 2026 21:57:54 -0400 Subject: [PATCH 7/7] Resolve lairs from PyPI lairs 0.5.0 is now published on PyPI, so drop the tool.uv.sources local-path redirect and the CI source-checkout steps; the plain lairs>=0.5.0 requirement resolves from PyPI for both local and CI installs. --- .github/workflows/ci.yml | 20 -------- pyproject.toml | 6 --- uv.lock | 98 ++-------------------------------------- 3 files changed, 5 insertions(+), 119 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffbf087..f18f968 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,15 +161,6 @@ jobs: steps: - uses: actions/checkout@v4 - # lairs 0.5.0 is not yet on PyPI; install it from its public source repo - # (mirrors the tool.uv.sources path used for local development). - - name: Check out lairs - uses: actions/checkout@v4 - with: - repository: layers-pub/lairs - ref: main - path: lairs - - name: Set up Python uses: actions/setup-python@v5 with: @@ -178,7 +169,6 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - pip install ./lairs pip install -e ".[dev,behavioral-analysis]" - name: Run pyright @@ -191,15 +181,6 @@ jobs: steps: - uses: actions/checkout@v4 - # lairs 0.5.0 is not yet on PyPI; install it from its public source repo - # (mirrors the tool.uv.sources path used for local development). - - name: Check out lairs - uses: actions/checkout@v4 - with: - repository: layers-pub/lairs - ref: main - path: lairs - - name: Install pnpm uses: pnpm/action-setup@v4 with: @@ -242,7 +223,6 @@ jobs: - name: Install dependencies in venv run: | .venv/bin/pip install --upgrade pip - .venv/bin/pip install ./lairs .venv/bin/pip install -e ".[dev,api,training,stats,behavioral-analysis]" - name: Set up test fixtures diff --git a/pyproject.toml b/pyproject.toml index aac963d..df967ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,9 +192,3 @@ exclude = [ dev = [ "hypothesis>=6.155.0", ] - -# lairs 0.5.0 (built on didactic 0.9 / panproto 0.56) is not yet on PyPI; resolve -# it from the local sibling checkout during development. uv ignores this source -# when bead is published, leaving the plain ``lairs>=0.5.0`` requirement. -[tool.uv.sources] -lairs = { path = "../lairs", editable = true } diff --git a/uv.lock b/uv.lock index 93ff938..03abc52 100644 --- a/uv.lock +++ b/uv.lock @@ -263,7 +263,7 @@ requires-dist = [ { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.155.0" }, { name = "jinja2", specifier = ">=3.0.0" }, { name = "krippendorff", specifier = ">=0.6.0" }, - { name = "lairs", editable = "../lairs" }, + { name = "lairs", specifier = ">=0.5.0" }, { name = "langcodes", specifier = ">=3.3.0" }, { name = "lark", specifier = ">=1.0.0" }, { name = "openai", marker = "extra == 'api'", specifier = ">=1.0.0" }, @@ -1043,7 +1043,7 @@ wheels = [ [[package]] name = "lairs" version = "0.5.0" -source = { editable = "../lairs" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "didactic" }, { name = "duckdb" }, @@ -1055,97 +1055,9 @@ dependencies = [ { name = "textual" }, { name = "websockets" }, ] - -[package.metadata] -requires-dist = [ - { name = "av", marker = "extra == 'video'" }, - { name = "brat", marker = "extra == 'brat'" }, - { name = "conllu", marker = "extra == 'conllu'" }, - { name = "datasets", marker = "extra == 'hf'" }, - { name = "decord", marker = "python_full_version < '3.14' and extra == 'video'" }, - { name = "didactic", specifier = ">=0.9.0" }, - { name = "duckdb" }, - { name = "fsspec", marker = "extra == 'fsspec'" }, - { name = "geopandas", marker = "extra == 'geo'" }, - { name = "glazing", marker = "extra == 'lexical'", specifier = ">=0.2.3" }, - { name = "habanero", marker = "extra == 'scholarly'" }, - { name = "httpx" }, - { name = "httpx", marker = "extra == 'appview'" }, - { name = "httpx", marker = "extra == 'reconciliation'" }, - { name = "huggingface-hub", marker = "extra == 'hf'" }, - { name = "label-studio-sdk", marker = "extra == 'labelstudio'" }, - { name = "lairs", extras = ["fsspec"], marker = "extra == 'ml'" }, - { name = "lairs", extras = ["hf"], marker = "extra == 'ml'" }, - { name = "lairs", extras = ["torch"], marker = "extra == 'ml'" }, - { name = "libipld" }, - { name = "librosa", marker = "extra == 'audio'" }, - { name = "mlflow", marker = "extra == 'tracking'" }, - { name = "mne", marker = "extra == 'neural'" }, - { name = "multiformats" }, - { name = "obonet", marker = "extra == 'bio'" }, - { name = "panproto", specifier = ">=0.56.0" }, - { name = "penman", marker = "extra == 'amr'" }, - { name = "polars", marker = "extra == 'polars'" }, - { name = "pyalex", marker = "extra == 'scholarly'" }, - { name = "pyarrow" }, - { name = "pympi-ling", marker = "extra == 'elan'" }, - { name = "qwikidata", marker = "extra == 'wikidata'" }, - { name = "shapely", marker = "extra == 'geo'" }, - { name = "soundfile", marker = "extra == 'audio'" }, - { name = "spacy", marker = "extra == 'spacy'" }, - { name = "sparqlwrapper", marker = "extra == 'wikidata'" }, - { name = "tensorflow", marker = "python_full_version < '3.14' and extra == 'tf'" }, - { name = "textgrid", marker = "extra == 'textgrid'" }, - { name = "textual", specifier = ">=8" }, - { name = "torch", marker = "extra == 'torch'" }, - { name = "wandb", marker = "extra == 'tracking'" }, - { name = "webdataset", marker = "extra == 'webdataset'" }, - { name = "websockets", specifier = ">=13" }, - { name = "xarray", marker = "extra == 'neural'" }, -] -provides-extras = ["hf", "torch", "tf", "webdataset", "polars", "fsspec", "conllu", "brat", "elan", "textgrid", "amr", "spacy", "labelstudio", "wikidata", "lexical", "reconciliation", "scholarly", "geo", "bio", "audio", "video", "neural", "tracking", "appview", "ml"] - -[package.metadata.requires-dev] -dev = [ - { name = "hypothesis" }, - { name = "lairs", extras = ["amr"] }, - { name = "lairs", extras = ["appview"] }, - { name = "lairs", extras = ["audio"] }, - { name = "lairs", extras = ["bio"] }, - { name = "lairs", extras = ["brat"] }, - { name = "lairs", extras = ["conllu"] }, - { name = "lairs", extras = ["elan"] }, - { name = "lairs", extras = ["fsspec"] }, - { name = "lairs", extras = ["geo"] }, - { name = "lairs", extras = ["hf"] }, - { name = "lairs", extras = ["lexical"] }, - { name = "lairs", extras = ["neural"] }, - { name = "lairs", extras = ["polars"] }, - { name = "lairs", extras = ["reconciliation"] }, - { name = "lairs", extras = ["scholarly"] }, - { name = "lairs", extras = ["spacy"] }, - { name = "lairs", extras = ["textgrid"] }, - { name = "lairs", extras = ["tf"] }, - { name = "lairs", extras = ["torch"] }, - { name = "lairs", extras = ["tracking"] }, - { name = "lairs", extras = ["video"] }, - { name = "lairs", extras = ["webdataset"] }, - { name = "lairs", extras = ["wikidata"] }, - { name = "numpy", specifier = ">=2.3" }, - { name = "pytest" }, - { name = "pytest-cov" }, - { name = "pytest-recording" }, - { name = "ruff" }, - { name = "ty" }, -] -docs = [ - { name = "griffe", specifier = ">=1.5" }, - { name = "mkdocs", specifier = ">=1.6" }, - { name = "mkdocs-autorefs", specifier = ">=1.2" }, - { name = "mkdocs-cinder", specifier = ">=1.2" }, - { name = "mkdocstrings", extras = ["python"], specifier = ">=0.27" }, - { name = "pygments", specifier = ">=2.18" }, - { name = "pymdown-extensions", specifier = ">=10.7" }, +sdist = { url = "https://files.pythonhosted.org/packages/c1/e3/638d8889ce32c3a296ee077788e80d5a8a8dc73bb8749e810a0c91e5a4fb/lairs-0.5.0.tar.gz", hash = "sha256:fb01df67e4cd47b2dbbb2e8604550413dbffbe14783a062004843e5af51c21e0", size = 718022, upload-time = "2026-06-30T01:17:05.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/50/77b73f0d065ab94fb4f7fa6a96ad489835ca01f543a3c2b2b726108bcce6/lairs-0.5.0-py3-none-any.whl", hash = "sha256:b49d94297e47c61c073b880049a84053588d06320b6b351522329855ebe8f1c5", size = 431602, upload-time = "2026-06-30T01:17:03.263Z" }, ] [[package]]