Add e2e tests

.github/workflows/python-package.yml

@@ -11,7 +11,6 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -41,3 +40,36 @@ jobs:
     - name: Test Build
       run: |
         poetry build
+
+  test-e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.13"]
+        tf-version:
+          - "opentofu@v1.10.5"
+          - "opentofu@v1.9.3"
+        e2e-project:
+          - "mathprovider"
+    needs: build
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+    - name: Install dependencies
+      working-directory: e2e/${{ matrix.e2e-project }}
+      run: |
+        poetry install
+    - name: Install ${{ matrix.tf-version }}
+      working-directory: e2e/${{ matrix.e2e-project }}
+      run: |
+        poetry run e2e-tool install ${{matrix.tf-version}}
+    - name: Run E2E Tests
+      working-directory: e2e/${{ matrix.e2e-project }}
+      run: |
+        poetry run e2e-tool run --tf ${{matrix.tf-version}}

.pyre_configuration

@@ -3,6 +3,10 @@
   "source_directories": [
     "."
   ],
+  "search_path": [
+    "./e2e/e2e_framework",
+    "./e2e/mathprovider"
+  ],
   "ignore_all_errors": [
     "tf/gen/*"
   ]

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - **Descriptions and Deprecation Fields for Schemas and Blocks**:
   - Added `description`, `description_kind`, and `deprecated` fields to Schema and Block. This allows resource, data sources, and providers to specify these fields for documentation.
+- **End-to-End Tests**:
+  - Added comprehensive end-to-end tests against OpenTofu. This ensures that providers, resources, and data sources work correctly against the real OpenTofu CLI.
 
 ## 1.1.0
 

Makefile

@@ -8,12 +8,12 @@ prepare-venv:
 
 format:
 	# Format and sort imports with ruff
-	$(HIDE)$(POETRY) run ruff format $(MODULE)
-	$(HIDE)$(POETRY) run ruff check --fix $(MODULE)
+	$(HIDE)$(POETRY) run ruff format $(MODULE) e2e
+	$(HIDE)$(POETRY) run ruff check --fix $(MODULE) e2e
 
 test-format:
-	$(HIDE)$(POETRY) run ruff format $(MODULE) --check
-	$(HIDE)$(POETRY) run ruff check $(MODULE)
+	$(HIDE)$(POETRY) run ruff format $(MODULE) e2e --check
+	$(HIDE)$(POETRY) run ruff check $(MODULE) e2e
 
 update-tfplugin-proto:
 	$(HIDE)curl https://raw.githubusercontent.com/opentofu/opentofu/main/docs/plugin-protocol/$(TFPLUGIN_PROTO) > tfplugin.proto

e2e/e2e_framework/e2e_framework/__init__.py

@@ -0,0 +1 @@
+from e2e_framework.framework import ProviderTest as ProviderTest

e2e/e2e_framework/e2e_framework/cli.py

@@ -0,0 +1,136 @@
+import argparse
+import io
+import os
+import platform
+import shutil
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+import requests
+
+DEFAULT_TF_PACKAGE = "opentofu@v1.10.5"
+
+
+def main():
+    handlers = {
+        "install": cmd_install,
+        "run": cmd_run,
+    }
+
+    parser = argparse.ArgumentParser(description="TF e2e tool")
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    install_parser = subparsers.add_parser("install", help="Install OpenTofu into test environment")
+    install_parser.add_argument("package_name", help="Version to install eg: opentofu@v1.2.3")
+
+    run_parser = subparsers.add_parser("run", help="Run tests")
+    run_parser.add_argument("--tf", dest="package_name", help="Version to run against eg: opentofu@v1.2.3")
+    run_parser.add_argument("tests", nargs="*", help="Test modules, classes or methods to run (default: cwd)")
+    run_parser.add_argument("-v", action="count", default=1, help="Verbosity level")
+
+    matches = parser.parse_args(sys.argv[1:])
+    handlers[matches.command](matches)
+
+
+def cmd_run(args: argparse.Namespace):
+    binary = tf_package_to_binary(args.package_name or DEFAULT_TF_PACKAGE)
+
+    if not binary.exists():
+        print("Error: OpenTofu binary not installed! Please run 'install' command first.", file=sys.stderr)
+        sys.exit(1)
+
+    if args.tests:
+        tests = unittest.defaultTestLoader.loadTestsFromNames(args.tests)
+    else:
+        tests = unittest.defaultTestLoader.discover(
+            start_dir=os.getcwd(),
+            pattern="test_*.py",
+        )
+
+    os.environ["TF_BINARY_NAME"] = str(binary)
+
+    runner = unittest.TextTestRunner(verbosity=args.v)
+    result = runner.run(tests)
+
+    if not result.wasSuccessful():
+        sys.exit(1)
+
+
+def tf_package_to_binary(package: str) -> Path:
+    if not package.startswith("opentofu@"):
+        print("Error: package name must start with 'opentofu@' -- only OpenTofu is supported", file=sys.stderr)
+        sys.exit(1)
+
+    version = package.split("@", 1)[1]
+    binary_name = f"tofu-{version}"
+    return Path(sys.executable).parent / binary_name
+
+
+def cmd_install(args: argparse.Namespace):
+    package = args.package_name
+    destination_path = tf_package_to_binary(package)
+
+    if destination_path.exists():
+        print(f"OpenTofu package {package} is already installed into the venv!")
+        return
+
+    plat = sys.platform.lower()
+    arch = {"x86_64": "amd64"}.get(platform.machine().lower(), platform.machine().lower())
+
+    version = package.split("@", 1)[1]
+    zip_name = f"tofu_{version.lstrip('v')}_{plat}_{arch}.zip"
+
+    print(f"Fetching release {version}...")
+    release = get_tofu_release(version)
+
+    assets = [a for a in release["assets"] if a["name"] == zip_name]
+
+    if not assets:
+        print(f"Error: No release asset found for {zip_name}", file=sys.stderr)
+        sys.exit(1)
+
+    asset = assets[0]
+
+    print(f"Downloading {zip_name}...")
+
+    zip_request = requests.get(asset["browser_download_url"])
+    if zip_request.status_code != 200:
+        print(
+            f"Error: Failed to download asset {zip_name}: {zip_request.status_code} {zip_request.text}", file=sys.stderr
+        )
+        sys.exit(1)
+
+    zip_contents = io.BytesIO(zip_request.content)
+    with zipfile.ZipFile(zip_contents) as zf:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            zf.extract("tofu", path=tmpdir)
+            shutil.move(f"{tmpdir}/tofu", destination_path)
+            destination_path.chmod(0o755)
+
+    print(f"Installed OpenTofu {version} to {destination_path}")
+
+
+def get_tofu_release(tag) -> dict:
+    """
+    curl -L \
+  -H "Accept: application/vnd.github+json" \
+  -H "Authorization: Bearer <YOUR-TOKEN>" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  https://api.github.com/repos/OWNER/REPO/releases/tags/TAG
+    """
+
+    release = requests.get(
+        f"https://api.github.com/repos/opentofu/opentofu/releases/tags/{tag}",
+        headers={
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+
+    if release.status_code != 200:
+        raise ValueError(f"Failed to get release info for tag {tag}: {release.status_code} {release.text}")
+
+    return release.json()

e2e/e2e_framework/e2e_framework/framework.py

@@ -0,0 +1,127 @@
+import os
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from textwrap import dedent
+from typing import Optional, Sequence
+from unittest import TestCase
+
+
+@dataclass
+class Result:
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+class ProviderTest(TestCase):
+    PROVIDER_NAME = ""
+
+    def setUp(self):
+        super().setUp()
+        tmp = tempfile.TemporaryDirectory()
+        self.tmp = Path(tmp.name)
+        self.addCleanup(tmp.cleanup)
+        self.rc_file = self.tmp / ".tf.rc"
+
+    def _get_tf_command(self):
+        return os.environ.get("TF_BINARY_NAME", "tofu")
+
+    def run_test(self, hcl: str, expect_error: bool = False, error_message: str = ""):
+        tf_file = self.tmp / "main.tf"
+        tf_file.write_text(hcl)
+
+    def _tf_run(
+        self, command: Sequence[str], hcl: str, expect_error=False, expect_in_output: Optional[Sequence[str]] = None
+    ) -> Result:
+        (self.tmp / "main.tf").write_text(hcl)
+        self._auto_install_rc()
+
+        (self.tmp / "version.tf").write_text(
+            dedent(
+                """\
+            terraform {
+              required_providers {
+                %s = {
+                  source  = "%s"
+                }
+              }
+            }
+        """
+                % (
+                    self.PROVIDER_NAME.split("/")[-1],
+                    self.PROVIDER_NAME,
+                )
+            )
+        )
+
+        env = {
+            "TF_CLI_CONFIG_FILE": str(self.rc_file),
+            "TF_CLI_ARGS": "-no-color",
+        }
+
+        invoke_args = [self._get_tf_command()] + list(command)
+
+        process = subprocess.Popen(
+            invoke_args,
+            cwd=self.tmp,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+        )
+        stdout, stderr = process.communicate()
+        res = Result(
+            returncode=process.returncode,
+            stdout=stdout.decode(),
+            stderr=stderr.decode(),
+        )
+
+        if expect_error:
+            self.assertNotEqual(res.returncode, 0)
+        else:
+            self.assertEqual(res.returncode, 0, msg=res.stderr)
+
+        for s in expect_in_output or []:
+            self.assertIn(s, res.stdout + res.stderr)
+
+        return res
+
+    def tf_plan(self, hcl: str, expect_error=False, expect_in_output: Optional[Sequence[str]] = None) -> Result:
+        return self._tf_run(["plan"], hcl=hcl, expect_error=expect_error, expect_in_output=expect_in_output)
+
+    def tf_apply(self, hcl: str, expect_error=False, expect_in_output: Optional[Sequence[str]] = None) -> Result:
+        return self._tf_run(
+            ["apply", "-auto-approve"], hcl=hcl, expect_error=expect_error, expect_in_output=expect_in_output
+        )
+
+    def _auto_install_rc(self):
+        if not self.rc_file.exists():
+            self.rc_file.write_text(
+                dedent(
+                    """\
+                provider_installation {
+                  dev_overrides {
+                      "%s" = "%s"
+                  }
+                  direct {}
+                }
+            """
+                    % (self.PROVIDER_NAME, str(Path(sys.executable).parent))
+                )
+            )
+
+    def tf_state(self) -> dict:
+        process = subprocess.Popen(
+            [self._get_tf_command(), "show", "-json"],
+            cwd=self.tmp,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env={"TF_CLI_CONFIG_FILE": str(self.rc_file), "TF_CLI_ARGS": "-no-color"},
+        )
+        stdout, stderr = process.communicate()
+        self.assertEqual(process.returncode, 0, msg=stderr.decode())
+        import json
+
+        return json.loads(stdout.decode())