From ea0e47b1e3adc198e7e024775888bfeea859607e Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Thu, 17 Jul 2025 15:55:52 +0100
Subject: [PATCH 01/19] Added ability to import config defined fbi_index and
 fbi_annotations for elasticsearch. If they aren't defined then the hard coded
 values will be used instead.

---
 fbi_core/annotate.py  |  7 +++++--
 fbi_core/conf.py      | 10 ++++++++++
 fbi_core/fbi_tools.py |  7 +++++--
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/fbi_core/annotate.py b/fbi_core/annotate.py
index fc0329f..3e057f3 100644
--- a/fbi_core/annotate.py
+++ b/fbi_core/annotate.py
@@ -8,14 +8,17 @@
 import requests
 from ceda_es_client import CEDAElasticsearchClient
 
-from .conf import APIKEY
+from .conf import APIKEY, es_annotation 
 
 if APIKEY:
     es = CEDAElasticsearchClient(headers={"x-api-key": APIKEY})
 else:
     es = CEDAElasticsearchClient()
 
-indexname = "fbi-annotations"
+if es_annotation:
+    indexname = es_annotation
+else:
+    indexname = "fbi-annotations"
 
 
 def get_moles_records():
diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index d1ef3f5..881153d 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -6,6 +6,16 @@
 if os.path.exists(conf_file):
     conf = yaml.load(open(conf_file), Loader=yaml.Loader)
     APIKEY = conf["ES"]["api_key"]
+
+    if "index_fbi" in conf["ES"]:
+        es_index = conf["ES"]["index_fbi"]
+    else:
+        es_index = None
+
+    if "index_fbi_annotation" in conf["ES"]:
+        es_annotation = conf["ES"]["index_fbi_annotation"]
+    else:
+        es_annotation = None
 else:
     APIKEY = None
 
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index e9ad6bc..c892666 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -8,14 +8,17 @@
 from ceda_es_client import CEDAElasticsearchClient
 from elasticsearch.helpers import scan
 
-from .conf import APIKEY
+from .conf import APIKEY, es_index
 
 if APIKEY:
     es = CEDAElasticsearchClient(headers={"x-api-key": APIKEY})
 else:
     es = CEDAElasticsearchClient()
 
-indexname = "fbi-2022"
+if es_index:
+    indexname = es_index
+else:
+    indexname = "fbi-2022"
 
 
 def fbi_records(

From a0d80bc481ec8299699c0fea52d3fb4f56b95abc Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Tue, 29 Jul 2025 13:43:20 +0100
Subject: [PATCH 02/19] Removed ceda-elasticsearch-client as it isn't needed
 and is making updating difficult.

Switched cedaelasticsearch client with Elasticsearch in annotate.py and fbi_tools.py, added es_host to conf.py and the other two so elasticsearch uses CEDA loadbalancer as host.
---
 fbi_core/annotate.py  | 8 ++++----
 fbi_core/conf.py      | 1 +
 fbi_core/fbi_tools.py | 9 ++++-----
 requirements.txt      | 1 -
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/fbi_core/annotate.py b/fbi_core/annotate.py
index 3e057f3..eadad9b 100644
--- a/fbi_core/annotate.py
+++ b/fbi_core/annotate.py
@@ -5,15 +5,15 @@
 from collections import defaultdict
 
 import elasticsearch
+from elasticsearch import Elasticsearch
 import requests
-from ceda_es_client import CEDAElasticsearchClient
 
-from .conf import APIKEY, es_annotation 
+from .conf import APIKEY, es_annotation, host_es
 
 if APIKEY:
-    es = CEDAElasticsearchClient(headers={"x-api-key": APIKEY})
+    es = Elasticsearch(host_es, headers={"x-api-key": APIKEY})
 else:
-    es = CEDAElasticsearchClient()
+    es = Elasticsearch()
 
 if es_annotation:
     indexname = es_annotation
diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index 881153d..6b9f14c 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -6,6 +6,7 @@
 if os.path.exists(conf_file):
     conf = yaml.load(open(conf_file), Loader=yaml.Loader)
     APIKEY = conf["ES"]["api_key"]
+    host_es = conf["ES"]["host"]
 
     if "index_fbi" in conf["ES"]:
         es_index = conf["ES"]["index_fbi"]
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index c892666..6628e1e 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -4,16 +4,15 @@
 import re
 from datetime import datetime
 
-import elasticsearch
-from ceda_es_client import CEDAElasticsearchClient
+from elasticsearch import Elasticsearch
 from elasticsearch.helpers import scan
 
-from .conf import APIKEY, es_index
+from .conf import APIKEY, es_index, host_es
 
 if APIKEY:
-    es = CEDAElasticsearchClient(headers={"x-api-key": APIKEY})
+    es = Elasticsearch(host_es, headers={"x-api-key": APIKEY})
 else:
-    es = CEDAElasticsearchClient()
+    es = Elasticsearch()
 
 if es_index:
     indexname = es_index
diff --git a/requirements.txt b/requirements.txt
index f71b128..99e2840 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-ceda-elasticsearch-client
 tabulate
 click
 pyyaml

From 080703be6fd77e3ed3773c55119c75a20f65eb1e Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Tue, 29 Jul 2025 14:21:00 +0100
Subject: [PATCH 03/19] Changed how conf.py worked, function call rather than
 passing variables in using from.

Amended annotate.py and fbi_tools.py to use conf.py
---
 fbi_core/annotate.py  |  8 +++++---
 fbi_core/conf.py      | 30 +++++++++++++++++-------------
 fbi_core/fbi_tools.py |  8 +++++---
 3 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/fbi_core/annotate.py b/fbi_core/annotate.py
index eadad9b..4237538 100644
--- a/fbi_core/annotate.py
+++ b/fbi_core/annotate.py
@@ -8,10 +8,12 @@
 from elasticsearch import Elasticsearch
 import requests
 
-from .conf import APIKEY, es_annotation, host_es
+from .conf import load_config
 
-if APIKEY:
-    es = Elasticsearch(host_es, headers={"x-api-key": APIKEY})
+api_key, host_es, es_index, es_annotation = load_config()
+
+if api_key:
+    es = Elasticsearch(host_es, headers={"x-api-key": api_key})
 else:
     es = Elasticsearch()
 
diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index 6b9f14c..7cd7da9 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -3,20 +3,24 @@
 
 
 conf_file = os.path.join(os.environ["HOME"], ".fbi.yml")
-if os.path.exists(conf_file):
-    conf = yaml.load(open(conf_file), Loader=yaml.Loader)
-    APIKEY = conf["ES"]["api_key"]
-    host_es = conf["ES"]["host"]
 
-    if "index_fbi" in conf["ES"]:
-        es_index = conf["ES"]["index_fbi"]
-    else:
-        es_index = None
+def load_config():
+    if os.path.exists(conf_file):
+        conf = yaml.load(open(conf_file), Loader=yaml.Loader)
+        api_key = conf["ES"]["api_key"]
+        host_es = conf["ES"]["host"]
+
+        if "index_fbi" in conf["ES"]:
+            es_index = conf["ES"]["index_fbi"]
+        else:
+            es_index = None
 
-    if "index_fbi_annotation" in conf["ES"]:
-        es_annotation = conf["ES"]["index_fbi_annotation"]
+        if "index_fbi_annotation" in conf["ES"]:
+            es_annotation = conf["ES"]["index_fbi_annotation"]
+        else:
+            es_annotation = None
     else:
-        es_annotation = None
-else:
-    APIKEY = None
+        api_key = None
+
+    return api_key, host_es, es_index, es_annotation
 
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index 6628e1e..2a8db29 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -7,10 +7,12 @@
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import scan
 
-from .conf import APIKEY, es_index, host_es
+from .conf import load_config
 
-if APIKEY:
-    es = Elasticsearch(host_es, headers={"x-api-key": APIKEY})
+api_key, host_es, es_index, es_annotation = load_config()
+
+if api_key:
+    es = Elasticsearch(host_es, headers={"x-api-key": api_key})
 else:
     es = Elasticsearch()
 

From 650df60bb1c3a5c03af3efe80566df1e2916907c Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Tue, 29 Jul 2025 14:59:40 +0100
Subject: [PATCH 04/19] Further work to make fbi-core work with conf.py being a
 function rather than importing constants. .conf to conf in annotate.py and
 fbi_tools.py

Updated requirements.txt and setup.py to include missing requests package.

Updated conf.py to allow debugging in windows (if os.environ("Home") doesn't exists try except statement, won't work properly in MacOS or Linux.
---
 .gitignore            | 1 +
 fbi_core/annotate.py  | 2 +-
 fbi_core/conf.py      | 7 ++++---
 fbi_core/fbi_tools.py | 2 +-
 requirements.txt      | 1 +
 setup.py              | 2 +-
 6 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5e5c3f9..d336147 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ build
 *.egg-info
 dist
 dump_output
+/home/
\ No newline at end of file
diff --git a/fbi_core/annotate.py b/fbi_core/annotate.py
index 4237538..a4c00b9 100644
--- a/fbi_core/annotate.py
+++ b/fbi_core/annotate.py
@@ -8,7 +8,7 @@
 from elasticsearch import Elasticsearch
 import requests
 
-from .conf import load_config
+from conf import load_config
 
 api_key, host_es, es_index, es_annotation = load_config()
 
diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index 7cd7da9..c850c31 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -1,10 +1,11 @@
 import yaml
 import os
 
-
-conf_file = os.path.join(os.environ["HOME"], ".fbi.yml")
-
 def load_config():
+    try:
+        conf_file = os.path.join(os.environ["HOME"], ".fbi.yml") # for prod in Linux based environment
+    except:
+        conf_file = os.path.join("./home/", ".fbi.yml") # For testing in Windows environment, won't work in Linux/ Mac OS as "HOME" env variable exists. So another solution for debugging would be required.
     if os.path.exists(conf_file):
         conf = yaml.load(open(conf_file), Loader=yaml.Loader)
         api_key = conf["ES"]["api_key"]
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index 2a8db29..07871a8 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -7,7 +7,7 @@
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import scan
 
-from .conf import load_config
+from conf import load_config
 
 api_key, host_es, es_index, es_annotation = load_config()
 
diff --git a/requirements.txt b/requirements.txt
index 99e2840..bc36dd3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ click
 pyyaml
 elasticsearch
 colorama
+requests
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 79afcc7..c381ed5 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@
     author_email='sam.pepler@stfc.ac.uk',
     # Choose your license
     license='BSD',
-    install_requires=['tabulate', 'click', 'pyyaml', 'elasticsearch', 'colorama'],
+    install_requires=['tabulate', 'click', 'pyyaml', 'elasticsearch', 'colorama', 'requests'],
 
     # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[

From 9f632d73a014a6de55c54c06787d10b7d60ffbb2 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Tue, 29 Jul 2025 15:14:05 +0100
Subject: [PATCH 05/19] Continuing to try and get fbi_indexer working in
 Kubernetes through stocktake installation.

---
 fbi_core/annotate.py  | 2 +-
 fbi_core/fbi_tools.py | 2 +-
 setup.py              | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/fbi_core/annotate.py b/fbi_core/annotate.py
index a4c00b9..4237538 100644
--- a/fbi_core/annotate.py
+++ b/fbi_core/annotate.py
@@ -8,7 +8,7 @@
 from elasticsearch import Elasticsearch
 import requests
 
-from conf import load_config
+from .conf import load_config
 
 api_key, host_es, es_index, es_annotation = load_config()
 
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index 07871a8..2a8db29 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -7,7 +7,7 @@
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import scan
 
-from conf import load_config
+from .conf import load_config
 
 api_key, host_es, es_index, es_annotation = load_config()
 
diff --git a/setup.py b/setup.py
index c381ed5..ec6a7c9 100644
--- a/setup.py
+++ b/setup.py
@@ -42,6 +42,7 @@
 
     entry_points={
         'console_scripts': [
+            'fbi_conf=fbi_core.conf:load_config'
             'fbi_filesize=fbi_core.fbi_filesize:summary',
             'fbi_ls=fbi_core.fbi_filesize:ls2',
             'fbi_random=fbi_core.fbi_filesize:random_paths',

From 6777e917eaa3e243164934dffbf4365c0056c17c Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Tue, 29 Jul 2025 15:15:32 +0100
Subject: [PATCH 06/19] Missed a comma in setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ec6a7c9..435a106 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@
 
     entry_points={
         'console_scripts': [
-            'fbi_conf=fbi_core.conf:load_config'
+            'fbi_conf=fbi_core.conf:load_config',
             'fbi_filesize=fbi_core.fbi_filesize:summary',
             'fbi_ls=fbi_core.fbi_filesize:ls2',
             'fbi_random=fbi_core.fbi_filesize:random_paths',

From 14cb2820f5accc06ffed81e0774530d7eba72ed7 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Tue, 29 Jul 2025 16:15:19 +0100
Subject: [PATCH 07/19] Added host_es, es_index and es_annotation definitions
 if the conf_file cannot be found to conf.py

Added elasticsearch to fbi_tools.py as it was accidentally overwritten by "from elasticsearch Import Elasticsearch" (which is also neeed)
---
 fbi_core/conf.py      | 3 +++
 fbi_core/fbi_tools.py | 1 +
 2 files changed, 4 insertions(+)

diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index c850c31..8f96392 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -22,6 +22,9 @@ def load_config():
             es_annotation = None
     else:
         api_key = None
+        host_es = None
+        es_index = None
+        es_annotation = None
 
     return api_key, host_es, es_index, es_annotation
 
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index 2a8db29..2ba45f5 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -4,6 +4,7 @@
 import re
 from datetime import datetime
 
+import elasticsearch
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import scan
 

From feb0b045afa9b791d7d7bdbffb19035d65d59810 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Wed, 30 Jul 2025 08:36:39 +0100
Subject: [PATCH 08/19] Very quick adjustment of requirements.txt file to pin
 elasticsearch to 8.12.0.

requirements.txt needs properly sorting, or simply replacing with pyproject.toml.
---
 requirements.txt | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index bc36dd3..41d7dbe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 tabulate
 click
 pyyaml
-elasticsearch
+elasticsearch = "8.12.0"
 colorama
 requests
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 435a106..871bd0a 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,7 @@
         'Intended Audience :: Developers',
         # Pick your license as you wish (should match "license" above)
         'License :: OSI Approved :: BSD License',
-        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.11',
     ],
     keywords='ingest',
     # You can just specify the packages manually here if your project is

From a250837b9807e74d019aae1bb341dcb33fd4fcaa Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Wed, 30 Jul 2025 09:15:16 +0100
Subject: [PATCH 09/19] Added pyproject.toml and poetry.lock to try and stop
 elasticsearch 9 being installed.

---
 .gitignore       |  2 +-
 poetry.lock      |  7 ++++++
 pyproject.toml   | 33 +++++++++++++++++++++++++
 requirements.txt |  6 -----
 setup.py         | 64 ------------------------------------------------
 x                |  0
 6 files changed, 41 insertions(+), 71 deletions(-)
 create mode 100644 poetry.lock
 create mode 100644 pyproject.toml
 delete mode 100644 requirements.txt
 delete mode 100644 setup.py
 delete mode 100644 x

diff --git a/.gitignore b/.gitignore
index d336147..30e5749 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,4 @@ build
 *.egg-info
 dist
 dump_output
-/home/
\ No newline at end of file
+/home/
diff --git a/poetry.lock b/poetry.lock
new file mode 100644
index 0000000..cf6da7c
--- /dev/null
+++ b/poetry.lock
@@ -0,0 +1,7 @@
+# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
+package = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = ">=3.11"
+content-hash = "dc7d249c43736b4dc5c67d29d4918f1ddf6778b40ca2b95bc8536b20175944eb"
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..db010b6
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,33 @@
+[project]
+name = "fbi-core"
+version = "0.1.0"
+description = "File Based Index (FBI) core queries. This module provides functions to interact with the CEDA FBI including read queries and methods to write records. Command line tools to list and summarise the archive are included here. A separate package, fbi-indexing-tools, provides tools to write records to the FBI. Documentation: https://cedadev.github.io/fbi-core/index.html"
+authors = [
+    {name = "Sam Pepler <sam.pepler@stfc.ac.uk> Matthew Paice <matthew.paice@stfc.ac.uk>"}
+]
+readme = "README.md"
+dynamic = ["dependencies"]
+requires-python = ">=3.11"
+
+[tool.poetry]
+requires-poetry = ">=2.0"
+
+packages = [
+    { include = "fbi_core"},
+]
+
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"
+
+dependencies = [
+    "tabulate (>=0.9.0,<0.10.0)",
+    "click (>=8.2.1,<9.0.0)",
+    "pyyaml (>=6.0.2,<7.0.0)",
+    "elasticsearch (==8.12.0)",
+    "colorama (>=0.4.6,<0.5.0)",
+    "requests (>=2.32.4,<3.0.0)"
+]
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 41d7dbe..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-tabulate
-click
-pyyaml
-elasticsearch = "8.12.0"
-colorama
-requests
\ No newline at end of file
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 871bd0a..0000000
--- a/setup.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# Always prefer setuptools over distutils
-# To use a consistent encoding
-from codecs import open
-from os import path
-
-from setuptools import find_packages, setup
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the relevant file
-with open(path.join(here, 'README.md'), encoding='utf-8') as f:
-    long_description = f.read()
-
-setup(
-    name='fbi-core',
-    version='1.3.1',
-    description='File Based Index  (FBI) core tools',
-    long_description=long_description,
-
-    # The project's main homepage.
-    url='http://www.ceda.ac.uk',
-    # Author details
-    author='Sam Pepler',
-    author_email='sam.pepler@stfc.ac.uk',
-    # Choose your license
-    license='BSD',
-    install_requires=['tabulate', 'click', 'pyyaml', 'elasticsearch', 'colorama', 'requests'],
-
-    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
-    classifiers=[
-        'Development Status :: 3 - Beta',
-        # Indicate who your project is intended for
-        'Intended Audience :: Developers',
-        # Pick your license as you wish (should match "license" above)
-        'License :: OSI Approved :: BSD License',
-        'Programming Language :: Python :: 3.11',
-    ],
-    keywords='ingest',
-    # You can just specify the packages manually here if your project is
-    # simple. Or you can use find_packages().
-    packages=find_packages(),
-
-    entry_points={
-        'console_scripts': [
-            'fbi_conf=fbi_core.conf:load_config',
-            'fbi_filesize=fbi_core.fbi_filesize:summary',
-            'fbi_ls=fbi_core.fbi_filesize:ls2',
-            'fbi_random=fbi_core.fbi_filesize:random_paths',
-            'fbi_show_record=fbi_core.fbi_filesize:show_record',
-            'fbi_parameters=fbi_core.fbi_filesize:show_parameters',
-            'fbi_last_updated=fbi_core.fbi_filesize:show_last_updated',
-            'fbi_split=fbi_core.fbi_filesize:find_splits',
-            'fbi_rsplit=fbi_core.fbi_filesize:find_rsplits',
-            'fbi_md5sum=fbi_core.fbi_filesize:md5sum',
-            'fbi_md5sum_check=fbi_core.fbi_filesize:check_archive_by_checksum',
-            'fbi_dump=fbi_core.fbi_dump:setup_run',
-            'fbi_batch_create=fbi_core.fbi_dump:setup_run',
-            'fbi_launch_run=fbi_core.fbi_dump:launch_run',
-            'fbi_batch_run=fbi_core.fbi_dump:batch_run',
-            'fbi_links_to=fbi_core.fbi_filesize:find_links_to',
-            'fbi_annotate=fbi_core.annotate:grab_moles',
-        ],
-    },
-)
diff --git a/x b/x
deleted file mode 100644
index e69de29..0000000

From ac22bc60f1de87eb9f93c24f09437c58c8d8a1a6 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Wed, 30 Jul 2025 09:23:15 +0100
Subject: [PATCH 10/19] Corrected versioning for fbi-core in pyproject.toml

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index db010b6..558582c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "fbi-core"
-version = "0.1.0"
+version = "1.3.2"
 description = "File Based Index (FBI) core queries. This module provides functions to interact with the CEDA FBI including read queries and methods to write records. Command line tools to list and summarise the archive are included here. A separate package, fbi-indexing-tools, provides tools to write records to the FBI. Documentation: https://cedadev.github.io/fbi-core/index.html"
 authors = [
     {name = "Sam Pepler <sam.pepler@stfc.ac.uk> Matthew Paice <matthew.paice@stfc.ac.uk>"}

From a0abc9d3dac48982219abfcc586050052ac03295 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Wed, 30 Jul 2025 09:53:08 +0100
Subject: [PATCH 11/19] Fixed pyproject.toml so that poetry.lock could be
 populated

---
 poetry.lock    | 315 ++++++++++++++++++++++++++++++++++++++++++++++++-
 pyproject.toml |   9 +-
 2 files changed, 315 insertions(+), 9 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index cf6da7c..6b4fe35 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,7 +1,318 @@
 # This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
-package = []
+
+[[package]]
+name = "certifi"
+version = "2025.7.14"
+description = "Python package for providing Mozilla's CA Bundle."
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2"},
+    {file = "certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995"},
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.4.2"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-win32.whl", hash = "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a"},
+    {file = "charset_normalizer-3.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a"},
+    {file = "charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c"},
+    {file = "charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-win32.whl", hash = "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7"},
+    {file = "charset_normalizer-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-win32.whl", hash = "sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58"},
+    {file = "charset_normalizer-3.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-win32.whl", hash = "sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7"},
+    {file = "charset_normalizer-3.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-win32.whl", hash = "sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471"},
+    {file = "charset_normalizer-3.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e"},
+    {file = "charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0"},
+    {file = "charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63"},
+]
+
+[[package]]
+name = "click"
+version = "8.2.1"
+description = "Composable command line interface toolkit"
+optional = false
+python-versions = ">=3.10"
+groups = ["main"]
+files = [
+    {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"},
+    {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+groups = ["main"]
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+
+[[package]]
+name = "elastic-transport"
+version = "8.17.1"
+description = "Transport classes and utilities shared among Python Elastic client libraries"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "elastic_transport-8.17.1-py3-none-any.whl", hash = "sha256:192718f498f1d10c5e9aa8b9cf32aed405e469a7f0e9d6a8923431dbb2c59fb8"},
+    {file = "elastic_transport-8.17.1.tar.gz", hash = "sha256:5edef32ac864dca8e2f0a613ef63491ee8d6b8cfb52881fa7313ba9290cac6d2"},
+]
+
+[package.dependencies]
+certifi = "*"
+urllib3 = ">=1.26.2,<3"
+
+[package.extras]
+develop = ["aiohttp", "furo", "httpx", "opentelemetry-api", "opentelemetry-sdk", "orjson", "pytest", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests", "respx", "sphinx (>2)", "sphinx-autodoc-typehints", "trustme"]
+
+[[package]]
+name = "elasticsearch"
+version = "8.12.0"
+description = "Python client for Elasticsearch"
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "elasticsearch-8.12.0-py3-none-any.whl", hash = "sha256:d394c5ce746bb8cb97827feae57759dae462bce34df221a6fdb6875c56476389"},
+    {file = "elasticsearch-8.12.0.tar.gz", hash = "sha256:58fd3876682f7529c33b9eeee701e71cfcc334bb45d725e315e22a0a5e2611fb"},
+]
+
+[package.dependencies]
+elastic-transport = ">=8,<9"
+
+[package.extras]
+async = ["aiohttp (>=3,<4)"]
+requests = ["requests (>=2.4.0,<3.0.0)"]
+
+[[package]]
+name = "idna"
+version = "3.10"
+description = "Internationalized Domain Names in Applications (IDNA)"
+optional = false
+python-versions = ">=3.6"
+groups = ["main"]
+files = [
+    {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"},
+    {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"},
+]
+
+[package.extras]
+all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
+
+[[package]]
+name = "pyyaml"
+version = "6.0.2"
+description = "YAML parser and emitter for Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
+    {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"},
+    {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"},
+    {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"},
+    {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"},
+    {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"},
+    {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"},
+    {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"},
+    {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"},
+    {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"},
+    {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"},
+    {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"},
+    {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"},
+    {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"},
+    {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"},
+    {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"},
+    {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"},
+    {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"},
+    {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"},
+    {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"},
+    {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"},
+    {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"},
+    {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"},
+    {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"},
+    {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"},
+    {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"},
+    {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"},
+    {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"},
+    {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"},
+    {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"},
+    {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"},
+    {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"},
+    {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"},
+    {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"},
+    {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"},
+]
+
+[[package]]
+name = "requests"
+version = "2.32.4"
+description = "Python HTTP for Humans."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"},
+    {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"},
+]
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset_normalizer = ">=2,<4"
+idna = ">=2.5,<4"
+urllib3 = ">=1.21.1,<3"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+
+[[package]]
+name = "tabulate"
+version = "0.9.0"
+description = "Pretty-print tabular data"
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"},
+    {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"},
+]
+
+[package.extras]
+widechars = ["wcwidth"]
+
+[[package]]
+name = "urllib3"
+version = "2.5.0"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+    {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"},
+    {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"},
+]
+
+[package.extras]
+brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""]
+h2 = ["h2 (>=4,<5)"]
+socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
+zstd = ["zstandard (>=0.18.0)"]
 
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.11"
-content-hash = "dc7d249c43736b4dc5c67d29d4918f1ddf6778b40ca2b95bc8536b20175944eb"
+content-hash = "36ce63aa546c1c18445a413caadc523d3ec132efd0a30ea8b4fc6a5444aa1b72"
diff --git a/pyproject.toml b/pyproject.toml
index 558582c..80b21db 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,23 +5,18 @@ description = "File Based Index (FBI) core queries. This module provides functio
 authors = [
     {name = "Sam Pepler <sam.pepler@stfc.ac.uk> Matthew Paice <matthew.paice@stfc.ac.uk>"}
 ]
+keywords = ['ingest']
 readme = "README.md"
 dynamic = ["dependencies"]
 requires-python = ">=3.11"
 
-[tool.poetry]
-requires-poetry = ">=2.0"
-
 packages = [
     { include = "fbi_core"},
 ]
 
-[tool.poetry.requires-plugins]
-poetry-plugin-export = ">=1.8"
-
 dependencies = [
-    "tabulate (>=0.9.0,<0.10.0)",
     "click (>=8.2.1,<9.0.0)",
+    "tabulate (>=0.9.0,<0.10.0)",
     "pyyaml (>=6.0.2,<7.0.0)",
     "elasticsearch (==8.12.0)",
     "colorama (>=0.4.6,<0.5.0)",

From f7ecf47aa9622c1eb8ec3c6391906f87d4db6896 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Thu, 14 Aug 2025 14:44:00 +0100
Subject: [PATCH 12/19] Initial fbi_flow_diagram with fbi index interaction
 points, especially fbi index field "location".

---
 fbi_loc_flow_diagram.md | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 fbi_loc_flow_diagram.md

diff --git a/fbi_loc_flow_diagram.md b/fbi_loc_flow_diagram.md
new file mode 100644
index 0000000..d06adb0
--- /dev/null
+++ b/fbi_loc_flow_diagram.md
@@ -0,0 +1,7 @@
+```mermaid
+---
+title: NLA flow diagram with fbi interactions, specifically location amendments
+---
+
+graph TB
+id1[Text in box]
\ No newline at end of file

From 49e40b2def1b8c1b18eacb290dc8e6ebd51fbe53 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Thu, 14 Aug 2025 15:46:15 +0100
Subject: [PATCH 13/19] Further updates to flow diagram

---
 fbi_loc_flow_diagram.md | 97 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 96 insertions(+), 1 deletion(-)

diff --git a/fbi_loc_flow_diagram.md b/fbi_loc_flow_diagram.md
index d06adb0..05c7808 100644
--- a/fbi_loc_flow_diagram.md
+++ b/fbi_loc_flow_diagram.md
@@ -4,4 +4,99 @@ title: NLA flow diagram with fbi interactions, specifically location amendments
 ---
 
 graph TB
-id1[Text in box]
\ No newline at end of file
+%% Disks
+external_data@{ shape: docs, label: "External data" }
+arrivals_service@{ shape: lin-cyl, label: "Arrivals Service \n Disk" }
+data_processing@{ shape: lin-cyl, label: "Datacentre Processing \n Disk" }
+CEDA_archive@{ shape: lin-cyl, label: "CEDA Archive \n Disk" }
+tape_archive@{ shape: lin-cyl, label: "CEDA Backup \n Tape" }
+cloud@{ shape: lin-cyl, label: "CEDA Cloud \n Cloud" }
+s3@{ shape: lin-cyl, label: "CEDA Object Store \n Amazon s3" }
+nla_cache@{ shape: lin-cyl, label: "NLA Cache \n Disk" }
+
+
+%% Databases
+archiveapp_db@{ shape: cyl, label: "cedaarchiveapp \n Database" }
+nla_control_db@{ shape: cyl, label: "NLA Control \n Database" }
+
+%% Elasticsearch / RabbitMQ
+fbi_index@{ shape: docs, label: "fbi-2022 \n Elasticsearch \n index" }
+rabbit_deposit@{ shape: docs, label: "f \n RabbitMQ" }
+
+%% Processes (inc repository where necessary)
+temp@{ shape: rect, label: "temp" }
+temp2@{ shape: rect, label: "temp" }
+temp3@{ shape: rect, label: "temp" }
+temp4@{ shape: rect, label: "temp" }
+temp5@{ shape: rect, label: "temp" }
+temp6@{ shape: rect, label: "temp" }
+temp7@{ shape: rect, label: "temp cloud" }
+temp8@{ shape: rect, label: "temp" }
+
+%%Flowchart general
+%% Archive to tape
+CEDA_archive --> temp6
+temp6 --> tape_archive
+
+%% Archive to cloud
+CEDA_archive --> temp7
+temp7 --> cloud
+
+%% Tape to NLA_cache
+tape_archive --> temp4
+temp4 --> nla_cache
+
+%% Archive to ObjectStore
+CEDA_archive --> temp8
+temp8 --> s3
+
+
+%% Flowchart storage
+subgraph storage[CEDA Storage]
+external_data -- "API" --> arrivals_service
+external_data -- "Received data" --> data_processing
+arrivals_service -- "scripts" --> CEDA_archive
+arrivals_service -- "scripts" --> data_processing
+data_processing -- "data scientists" --> CEDA_archive
+tape_archive
+cloud
+s3
+nla_cache
+end
+
+
+%% Servers/ k8s (need subgraphs for each of these with processes within them)
+subgraph deposit_server[Deposit Server]
+temp
+end
+
+subgraph ingest_machine[Ingest VM]
+temp2
+end
+
+subgraph ceda_aa[cedaarchiveapp VM]
+temp3
+end
+
+subgraph nla_vm[NLA VM]
+temp4
+end
+
+subgraph fbi_deposit[FBI Deposit k8s]
+temp5
+end
+
+subgraph backup[Backup VM]
+temp6
+end
+
+subgraph cloud_processing[Cloud Processing]
+temp7
+end
+
+subgraph s3_processing["Object Store (S3) Processing"]
+temp8
+end
+
+
+```
\ No newline at end of file

From 0dbf54951f04d766daee9b6b26bc44da61963204 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Thu, 23 Oct 2025 13:33:57 +0100
Subject: [PATCH 14/19] Added catalog-info for Backstage documentation to
 fbi-core

---
 catalog-info.yaml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 catalog-info.yaml

diff --git a/catalog-info.yaml b/catalog-info.yaml
new file mode 100644
index 0000000..d6b32b1
--- /dev/null
+++ b/catalog-info.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: fbi-core
+  description: "Core file based indexing (FBI) operations used by the NLA and other CEDA archiving systems."
+  tags:
+    - "priority:high"
+    - "visibility:private"
+    - "manager:sam-pepler"
+    - "deputy-manager:matthew-paice"
+    - "review:passed"
+spec:
+  type: library
+  owner: group:default/developers
+  lifecycle: production
+  system: system:default/nla
+  dependsOn:
+    - resource:default/elasticsearch-index-fbi
\ No newline at end of file

From 8f0ee07feb11ad7bdb33ef5101f9ad48069940d7 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Thu, 23 Oct 2025 14:01:12 +0100
Subject: [PATCH 15/19] Added elasticsearch-index-fbi to dependencies in
 catalog-info

---
 catalog-info.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalog-info.yaml b/catalog-info.yaml
index d6b32b1..0509b73 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -16,4 +16,5 @@ spec:
   lifecycle: production
   system: system:default/nla
   dependsOn:
-    - resource:default/elasticsearch-index-fbi
\ No newline at end of file
+    - resource:default/elasticsearch-index-fbi
+    - resource:default/elasticsearch-index-fbi-annotations
\ No newline at end of file

From 9e8ac0797a61d2423307412237c13939ee9df0fa Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Thu, 23 Oct 2025 14:06:11 +0100
Subject: [PATCH 16/19] Added fbi-annotations to fbi-core dependencies on
 backstage in catalog-info.yaml

---
 catalog-info.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalog-info.yaml b/catalog-info.yaml
index 0509b73..3415be9 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -17,4 +17,5 @@ spec:
   system: system:default/nla
   dependsOn:
     - resource:default/elasticsearch-index-fbi
-    - resource:default/elasticsearch-index-fbi-annotations
\ No newline at end of file
+    - resource:default/elasticsearch-index-fbi-annotations
+    - component:default/fbi-annotations
\ No newline at end of file

From b5c9b5a945198a64d22400d9db8a537f8f1f6dd1 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Mon, 20 Apr 2026 16:28:28 +0100
Subject: [PATCH 17/19] Updated conf.py (added requests and os), updated to
 Python 3.13 or greater in pyproject.toml and poetry.lock. Latest update to
 fbi_tools.py which just updates doc strings.

---
 fbi_core/conf.py      | 24 +++++++++++++++++++++++-
 fbi_core/fbi_tools.py |  6 +++---
 poetry.lock           |  4 ++--
 pyproject.toml        |  2 +-
 4 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index 8f96392..056f906 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -1,5 +1,6 @@
 import yaml
 import os
+from requests import get, codes
 
 def load_config():
     try:
@@ -26,5 +27,26 @@ def load_config():
         es_index = None
         es_annotation = None
 
-    return api_key, host_es, es_index, es_annotation
+
+    # Get spotlist on storage-d
+    spotlist_url = "https://cedaarchiveapp.ceda.ac.uk/storage-d/spotlist"
+    spots_page = get(spotlist_url)
+
+    if spots_page.status_code != codes['✓']:
+
+        print(spots_page.status_code + " Warning, error obtaining spotlist on storage-d  from https://cedaarchiveapp.ceda.ac.uk/storage-d/spotlist, using cached values from 05-Dec-2025")
+        try:
+            spotlist_file = os.path.join(os.environ["HOME"], "spotlist-cache.yml") # for prod in Linux based environment
+        except:
+            spotlist_file = os.path.join("./home/", "spotlist-cache.yml") # For testing in Windows environment, won't work in Linux/ Mac OS as "HOME" env variable exists. So another solution for debugging would be required.
+        if os.path.exists(spotlist_file):
+            spotlist = yaml.load(open(spotlist_file), Loader=yaml.Loader)
+            spotlist = spotlist["spotlist_cache"]
+
+        else:
+            spotlist = None
+    else:
+        spotlist = spots_page.text.splitlines()
+
+    return api_key, host_es, es_index, es_annotation, spotlist
 
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index 2ba45f5..51c6c2e 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -10,7 +10,7 @@
 
 from .conf import load_config
 
-api_key, host_es, es_index, es_annotation = load_config()
+api_key, host_es, es_index, es_annotation, spotlist = load_config()
 
 if api_key:
     es = Elasticsearch(host_es, headers={"x-api-key": api_key})
@@ -510,7 +510,7 @@ def fbi_listdir(
 
 
 def insert_item(record):
-    """Insert record by replaceing it"""
+    """Insert record by replacing it"""
     record_id = _create_id(record["path"])
     try:
         es.delete(index=indexname, id=record_id)
@@ -520,7 +520,7 @@ def insert_item(record):
 
 
 def update_item(record):
-    """Update a single document - overwrite feilds in record suplied."""
+    """Update a single document - overwrite fields in record suplied."""
     document = {"doc": record, "doc_as_upsert": True}
     es.update(
         index=indexname,
diff --git a/poetry.lock b/poetry.lock
index 6b4fe35..aea0ea5 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -314,5 +314,5 @@ zstd = ["zstandard (>=0.18.0)"]
 
 [metadata]
 lock-version = "2.1"
-python-versions = ">=3.11"
-content-hash = "36ce63aa546c1c18445a413caadc523d3ec132efd0a30ea8b4fc6a5444aa1b72"
+python-versions = ">=3.13"
+content-hash = "f4c71f0cc9a2536d9ac75d1ab5ee7314a54f02487ccfefdc1be50344e0114136"
diff --git a/pyproject.toml b/pyproject.toml
index 80b21db..6349d1b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ authors = [
 keywords = ['ingest']
 readme = "README.md"
 dynamic = ["dependencies"]
-requires-python = ">=3.11"
+requires-python = ">=3.13"
 
 packages = [
     { include = "fbi_core"},

From 87451c7515b2133ccb39a9c1e582c46ac0213e00 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Wed, 13 May 2026 15:05:06 +0100
Subject: [PATCH 18/19] Changed to new es basic_auth method

---
 fbi_core/annotate.py  |  9 ++++-----
 fbi_core/conf.py      | 22 ++++++++++++----------
 fbi_core/fbi_tools.py |  8 ++++----
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/fbi_core/annotate.py b/fbi_core/annotate.py
index be28cff..813e0e3 100644
--- a/fbi_core/annotate.py
+++ b/fbi_core/annotate.py
@@ -10,19 +10,18 @@
 
 from .conf import load_config
 
-api_key, host_es, es_index, es_annotation = load_config()
+user, password, host_es, es_index, es_annotation = load_config()
 
-if api_key:
-    es = Elasticsearch(host_es, headers={"x-api-key": api_key})
+if user and password:
+    es = Elasticsearch(host_es, basic_auth=(user, password), request_timeout=10)
 else:
-    es = Elasticsearch()
+    es = Elasticsearch(host_es)
 
 if es_annotation:
     indexname = es_annotation
 else:
     indexname = "fbi-annotations"
 
-
 def get_moles_records():
     """get moles info from catalogue"""
     obs_url = "https://catalogue.ceda.ac.uk/api/v2/observations.json"
diff --git a/fbi_core/conf.py b/fbi_core/conf.py
index 9244c27..0dbadf6 100644
--- a/fbi_core/conf.py
+++ b/fbi_core/conf.py
@@ -1,31 +1,33 @@
 import os
 from requests import get, codes
+import yaml
 
 def load_config():
     try:
         conf_file = os.path.join(os.environ["HOME"], ".fbi.yml") # for prod in Linux based environment
     except:
-        conf_file = os.path.join("./home/", ".fbi.yml") # For testing in Windows environment, won't work in Linux/ Mac OS as "HOME" env variable exists. So another solution for debugging would be required.
+        conf_file = os.path.join("./home/", ".fbi.yml") # For testing
     if os.path.exists(conf_file):
         conf = yaml.load(open(conf_file), Loader=yaml.Loader)
-        api_key = conf["ES"]["api_key"]
+        username = conf["ES"]["user"]
+        password = conf["ES"]["password"]
         host_es = conf["ES"]["host"]
 
         if "index_fbi" in conf["ES"]:
             es_index = conf["ES"]["index_fbi"]
         else:
-            es_index = None
+            es_index = "fbi-2022"
 
         if "index_fbi_annotation" in conf["ES"]:
             es_annotation = conf["ES"]["index_fbi_annotation"]
         else:
-            es_annotation = None
+            es_annotation = "fbi-annotations"
     else:
-        api_key = None
-        host_es = None
-        es_index = None
-        es_annotation = None
-
+        username = None
+        password = None
+        host_es = "https://elasticsearch.ceda.ac.uk:443"
+        es_index = "fbi-2022"
+        es_annotation = "fbi-annotations"
 
     # Get spotlist on storage-d
     spotlist_url = "https://cedaarchiveapp.ceda.ac.uk/storage-d/spotlist"
@@ -47,5 +49,5 @@ def load_config():
     else:
         spotlist = spots_page.text.splitlines()
 
-    return api_key, host_es, es_index, es_annotation, spotlist
+    return username, password, host_es, es_index, es_annotation, spotlist
 
diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index f49b4ec..f14e988 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -10,12 +10,12 @@
 
 from .conf import load_config
 
-api_key, host_es, es_index, es_annotation, spotlist = load_config()
+user, password, host_es, es_index, es_annotation, spotlist = load_config()
 
-if USERNAME:
-    es = elasticsearch.Elasticsearch(hosts=ES_HOSTS, basic_auth=(USERNAME, PASSWORD))
+if user and password:
+    es = elasticsearch.Elasticsearch(hosts=host_es, basic_auth=(user, password), request_timeout=10)
 else:
-    es = elasticsearch.Elasticsearch(hosts=ES_HOSTS)
+    es = elasticsearch.Elasticsearch(hosts=host_es)
 
 if es_index:
     indexname = es_index

From e18047bc21d83294c12baaf0ea8f8615822409c4 Mon Sep 17 00:00:00 2001
From: m-haines <matthew.r.haines@gmail.com>
Date: Wed, 10 Jun 2026 13:06:53 +0100
Subject: [PATCH 19/19] Changed to use user and password and not API key

---
 fbi_core/fbi_tools.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fbi_core/fbi_tools.py b/fbi_core/fbi_tools.py
index f14e988..f688ba1 100644
--- a/fbi_core/fbi_tools.py
+++ b/fbi_core/fbi_tools.py
@@ -13,9 +13,9 @@
 user, password, host_es, es_index, es_annotation, spotlist = load_config()
 
 if user and password:
-    es = elasticsearch.Elasticsearch(hosts=host_es, basic_auth=(user, password), request_timeout=10)
+    es = Elasticsearch(hosts=host_es, basic_auth=(user, password), request_timeout=10)
 else:
-    es = elasticsearch.Elasticsearch(hosts=host_es)
+    es = Elasticsearch(hosts=host_es)
 
 if es_index:
     indexname = es_index
@@ -455,7 +455,7 @@ def splits(path, batch_size=10000000, **kwargs):
 
 def make_dirs(directory):
     """
-    Make FBI records for a diretory and any missing parent directories.
+    Make FBI records for a directory and any missing parent directories.
 
     :param str directory: The directory to add.
     """