diff --git a/.pyre_configuration b/.pyre_configuration index bbc3350..1dbfc9a 100644 --- a/.pyre_configuration +++ b/.pyre_configuration @@ -8,6 +8,7 @@ "./e2e/mathprovider" ], "ignore_all_errors": [ - "tf/gen/*" + "tf/gen/*", + "docs/examples/*" ] } diff --git a/Makefile b/Makefile index 9ce49e8..8d2bde3 100644 --- a/Makefile +++ b/Makefile @@ -2,18 +2,19 @@ HIDE := @ TFPLUGIN_PROTO := tfplugin6.5.proto POETRY := poetry MODULE := tf +FORMATTABLE_SOURCES := $(MODULE) e2e docs/examples prepare-venv: $(HIDE)$(POETRY) install format: # Format and sort imports with ruff - $(HIDE)$(POETRY) run ruff format $(MODULE) e2e - $(HIDE)$(POETRY) run ruff check --fix $(MODULE) e2e + $(HIDE)$(POETRY) run ruff format $(FORMATTABLE_SOURCES) + $(HIDE)$(POETRY) run ruff check --fix $(FORMATTABLE_SOURCES) test-format: - $(HIDE)$(POETRY) run ruff format $(MODULE) e2e --check - $(HIDE)$(POETRY) run ruff check $(MODULE) e2e + $(HIDE)$(POETRY) run ruff format $(FORMATTABLE_SOURCES) --check + $(HIDE)$(POETRY) run ruff check $(FORMATTABLE_SOURCES) update-tfplugin-proto: $(HIDE)curl https://raw.githubusercontent.com/opentofu/opentofu/main/docs/plugin-protocol/$(TFPLUGIN_PROTO) > tfplugin.proto diff --git a/docs/caveats.md b/docs/caveats.md index cef74e9..269cc9d 100644 --- a/docs/caveats.md +++ b/docs/caveats.md @@ -13,8 +13,6 @@ _It's an uphill battle to get a Python provider into production._ If you want to write a provider in Python for production use, you will have to overcome these hurdles. It's doable, but **outside of scope of this package and you're on your own.** -You should very strongly consider only using this package for development, testing, and proof-of-concept work. - #### What's the big problem? The biggest problem is that Terraform **expects the provider to be a single binary**. @@ -80,7 +78,7 @@ This is a huge pain and an accident waiting to happen. Once you have single binaries ready for each OS/Arch, you need to generate hash sums, sign them, and generate a manifest. In theory, the official HashiCorp registry will allow you to upload these binaries and metadata using the standard Github release workflow. -However, I would strongly consider using a private registry or a different distribution method. +However, you should strongly consider using a private registry or a different distribution method. HashiCorp will almost certainly not officially approve and verify your provider for [the integration program](https://developer.hashicorp.com/terraform/docs/partnerships). You will have to set that up yourself. diff --git a/docs/elements.md b/docs/elements.md index 8266d57..4c62662 100644 --- a/docs/elements.md +++ b/docs/elements.md @@ -8,7 +8,7 @@ At a high level: * An _Element_ is either a [`DataSource`](api.html#data-sources) or a [`Resource`](api.html#resources). * An _Element_ has a _Schema_ that defines the attributes and blocks that it exposes to the user. * An _Attribute_ is a field name, a Type, and a set of behaviors. -* A _Type_ is a Python type that maps to a TF type. +* A _Type_ is a Python class that can convert between Python and TF representations of the underlying data type. ## Types @@ -73,13 +73,16 @@ A schema is a versioned collection of attributes and blocks that the element exp ## Errors -All errors are reporting using `Diagnostics`. +All errors are reported using `Diagnostics`. This parameter is passed into most operations, and you can add warnings or errors. Be aware: Operations that add error diagnostics will be considered failed by Terraform. Warnings are not, however. -You can add path information to your diagnostics. + +You may optionally add path information to your diagnostics. This allows TF to display which specific field led to the error. It's very helpful to the user. + +.. literalinclude:: examples/datasource-dns.py diff --git a/docs/examples/datasource-dns.py b/docs/examples/datasource-dns.py new file mode 100644 index 0000000..1c063fd --- /dev/null +++ b/docs/examples/datasource-dns.py @@ -0,0 +1,30 @@ +from typing import Optional + +import requests + +from tf.iface import Config, DataSource, ReadDataContext, State + + +class DnsResolver(DataSource): + ... + + def read(self, ctx: ReadDataContext, config: Config) -> Optional[State]: + hostname = config["hostname"] + typ = config.get("type", "A") + + resp = requests.get(f"https://dns.google/resolve?name={hostname}&type={typ}").json() + answer = resp.get("Answer") + + if not answer: + ctx.diagnostics.add_error( + summary="No DNS records found", + detail=f"No {typ} records found for {hostname}", + path=["hostname"], + ) + return None + + return { + **config, + "data": answer[0]["data"], + "ttl": answer[0]["TTL"], + } diff --git a/docs/examples/test-datasource-dns.py b/docs/examples/test-datasource-dns.py new file mode 100644 index 0000000..9d9ced0 --- /dev/null +++ b/docs/examples/test-datasource-dns.py @@ -0,0 +1,81 @@ +from unittest import TestCase + +import responses +from myprovider import DnsResolver, MyProvider + +from tf.iface import ReadDataContext +from tf.utils import Diagnostics, Unknown + + +class TestDnsResolver(TestCase): + def setUp(self): + super().setUp() + + self.responses = responses.RequestsMock() + self.responses.start() + self.addCleanup(self.responses.stop) + self.addCleanup(self.responses.reset) + + self.provider = MyProvider() + + def test_happy(self): + self.responses.add( + responses.GET, + "https://dns.google/resolve?name=example.com&type=A", + json={ + "Answer": [ + {"data": "1.2.3.4", "TTL": 123}, + ] + }, + ) + + dns_ds: DnsResolver = self.provider.new_data_source(DnsResolver) + context = ReadDataContext(Diagnostics(), self.provider.get_model_prefix() + dns_ds.get_name()) + + state = dns_ds.read( + context, + { + "hostname": "example.com", + "type": "A", + "data": Unknown, + "ttl": Unknown, + }, + ) + + self.assertFalse(context.diagnostics.has_errors()) + self.assertEqual( + { + "hostname": "example.com", + "type": "A", + "data": "1.2.3.4", + "ttl": 123, + }, + state, + ) + + def test_no_records(self): + self.responses.add( + responses.GET, + "https://dns.google/resolve?name=example.com&type=A", + json={}, + ) + + dns_ds = self.provider.new_data_source(DnsResolver) + context = ReadDataContext(Diagnostics(), self.provider.get_model_prefix() + dns_ds.get_name()) + + state = dns_ds.read( + context, + { + "hostname": "example.com", + "type": "A", + "data": Unknown, + "ttl": Unknown, + }, + ) + + self.assertIsNone(state) + self.assertTrue(context.diagnostics.has_errors()) + self.assertEqual(1, len(context.diagnostics.errors)) + self.assertEqual("No DNS records found", context.diagnostics.errors[0].summary) + self.assertEqual("No A records found for example.com", context.diagnostics.errors[0].detail) + self.assertEqual(["hostname"], context.diagnostics.errors[0].path) diff --git a/docs/index.rst b/docs/index.rst index c183aa0..7df1756 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ functions.md api.rst tutorial.rst + testing.rst example.md caveats.md tips_tricks.rst diff --git a/docs/testing.rst b/docs/testing.rst new file mode 100644 index 0000000..182d53b --- /dev/null +++ b/docs/testing.rst @@ -0,0 +1,20 @@ +Testing +======= + +The ``tf`` Framework treats element operations as pure functions. +An element operation takes a dictionary, does something to it, and returns a new dictionary. +The framework does not maintain its own state for element instances. + +Instead, the implementing provider is responsible for the "non-pure" parts of the operation such as making API calls to backend services. + +This separation of concerns lends itself well to unit testing. + +For example, suppose we have a DNS datasource. + +.. literalinclude:: examples/datasource-dns.py + +Then we might write tests like this: + +.. literalinclude:: examples/test-datasource-dns.py + +``tf`` consumers are encouraged to write test utilities to reduce boilerplate around diagnostic errors/warning assertions and ``Context`` construction. diff --git a/docs/tips_tricks.rst b/docs/tips_tricks.rst index 876bd71..211bc0e 100644 --- a/docs/tips_tricks.rst +++ b/docs/tips_tricks.rst @@ -5,8 +5,8 @@ Tips and Tricks Using a Debugger ================ -You pre-emptively launch your provider in a long-lived server mode by running ``terraform-provider-$providername --stable --dev`` in a terminal. -This will start the provider, dump a ``export TF_REATTACH_PROVIDERS=...`` command, and wait for connections. +To use a debugger, you must pre-emptively launch your provider in long-lived server mode by running ``terraform-provider-$providername --stable --dev`` in a terminal. +This will start the provider, dump an ``export TF_REATTACH_PROVIDERS=...`` command, and wait for connections. In another terminal, you can paste and execute the ``export TF_REATTACH_PROVIDERS=...`` command to set up your environment. Then you can run ``tofu plan`` or ``terraform apply`` as usual. @@ -35,7 +35,7 @@ The provider upload registry APIs are not standardized across hosting providers, Roughly, the steps to build a pex binary are: -#. Install pex: `pip install pex`, preferably into a separate virtual environment or as a development dependency of your provider project. +#. Install pex: ``pip install pex``, preferably into a separate virtual environment or as a development dependency of your provider project. #. Dump the dependencies of your provider into a requirements file: .. code-block:: shell diff --git a/docs/usage.md b/docs/usage.md index a6c850b..6f9ddf3 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -5,20 +5,18 @@ There are four primary interfaces in this framework: 1. **Provider** - By implementing this interface, you can define a new provider. This defines its own schema, and supplies resource, data source, and function classes to the framework. -1. **Data Source** - This interface is used to define a data source, which - is a read-only object that can be used to query information +1. **Data Source** - Defines a data source: a **read-only object** that can be used to query information from the provider or backing service. -1. **Resource** - This interface is used to define a resource, which - is a read-write object that can be used to create, update, +2. **Resource** - Defines a resource: a **read-write object** that can be used to create, update, and delete resources in the provider or backing service. Resources represent full "ownership" of the underlying object. This is the primary type you will use to interact with the system. -1. **Function** - This interface is used to define provider functions, which +3. **Function** - Defines provider functions, which are stateless operations that can transform data, perform calculations, or validate inputs. Functions are called directly from Terraform configurations and return a single value. -To use this interface, create one class implementing `Provider`, and any number +To use the `tf` framework, create one class implementing `Provider`, and any number of classes implementing `Resource`, `DataSource`, and `Function`. Then, call `run_provider` with an instance of your provider class. A basic @@ -39,8 +37,8 @@ def main(): ## Entry Point Name TF requires a specific naming convention for the provider. Your executable -must be named in the form of `terraform-provider-`. -This means that you must your [entrypoint](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) +must be named in the form of `terraform-provider-$providername`. +This means that you must name your [entrypoint](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) similarly. ```toml @@ -52,8 +50,9 @@ terraform-provider-myprovider = "mypackage.main:main" In order to get TF to use your provider, you must tell TF to run your provider from a custom path. -This is done by editing the `~/.terraformrc` or `~/.tofurc` file, -and setting the path to your virtual environment's `bin` directory (which contains the `terraform-provider-myprovider` script). +1. Create a new `.tofurc` file in your working directory. +1. Set the path to your virtual environment's `bin` directory (which contains the `terraform-provider-myprovider` script). +1. Run `export TOFU_CLI_CONFIG_FILE=$(pwd)/.tofurc` to tell OpenTofu to use this file for your current shell session. ```hcl provider_installation { @@ -65,6 +64,9 @@ provider_installation { } ``` +If you really, really want to have the developer-mode provider available globally, +you can put the same configuration in `~/.tofurc` or `~/.terraformrc` file. + ## Using the Provider Now you can use your provider in Terraform by specifying it in the `provider` block in your `main.tf`.