testcli.go

// Copyright (c) 2020 Jason T. Lenz.  All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.

// Package testcli is a helper utility for testing golang command line
// applications (CLI).  When using testcli, each CLI test exists within its own
// file system folder.  All test folders for a specific CLI are typically
// contained within a main folder which the testcli package "walks" entering all
// subdirs and executing each CLI test within a folder.  Test results are
// tracked and displayed using the golang standard testing infrastructure.
//
// Each test folder must contain the following text files:
//   * tCmd     : The CLI command to be executed including parameters and
//                options.  Within this file any '{{.cli}}' string is replaced
//                by the CLI being tested.  If a test folder is multiple levels
//                deep within the file tree the relative CLI path is adjusted
//                accordingly before executing tCmd.
//   * tStdout  : The expected stdout
//   * tStderr  : The expected sterr
//   * tExit    : The exit code
//
// The following are optional files:
//   * t*.check : Any file matching t*.check is compared directly against the
//                same filename t*.result.  This can be used to check the
//                output of any files generated by the CLI.
//
// Alternately, any of the output filenames above may end with "Regex" to do a
// regular expression match rather than a direct string comparison.  This can
// be useful to check output for which a portion changes often.  For example
// log file output that contains the date or time at the beginning of an output
// line.
//   * tStdoutRegex   : Contains a regular expression string to match against
//                      the expected stdout
//   * t*.checkRegex
//   * etc. ...
//
// The following file names are not used directly by testcli but are named as
// follows by convention if needed:
//   * tStdin   : Any text intended to be fed as stdin to the CLI.  Typically
//                this is accomplished within tCmd such as "{{.cli}} < tStdin"
//
// Note that test folder names serve as the test description and are displayed
// when a test fails or when using "go test -v".  Also note that folders can be
// nested as many levels deep as desired to categorize and group tests.
//
// A full example demontrating the use of testcli is contained within the
// "tests" folder of this package.
package testcli

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"testing"
)

const (
	cliReplace = "{{.cli}}"
	checkExt   = ".check"
	resultExt  = ".result"
)

// RunTests iterates through all of the tests at or below the current path
// using the specified CLI.  The CLI path can be absolute or relative.
func RunTests(t *testing.T, cliPath string) {
	// tCmd is the filename which contains a test
	const tCmd = "tCmd"
	// Check that command to be tested exists.
	cliAbs, err := filepath.Abs(cliPath)
	if err != nil {
		panic("Unable to convert relative path to absolute")
	}
	_, err = exec.LookPath(cliAbs)
	if err != nil {
		panic(cliPath + " does not exist or is not executable")
	}
	// Get system shell used to execute scripts
	shell, ok := os.LookupEnv("SHELL")
	if !ok {
		panic("Unable to identify shell to execute test command")
	}
	// Walk dir tree looking for tCmd files
	err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if filepath.Base(path) == tCmd {
			testPathAbs, err := filepath.Abs(filepath.Dir(path))
			if err != nil {
				panic("Unable to get absolute test path")
			}
			cliRel, err := filepath.Rel(testPathAbs, cliAbs)
			if err != nil {
				panic("Unable to get relative command line path")
			}
			err = runOneTest(t, shell, cliRel, path)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		panic("Unable to walk directory tree")
	}
}

func runOneTest(t *testing.T, shell, cliRel, testFile string) error {
	testPath := filepath.Dir(testFile)
	t.Run(testPath, func(t *testing.T) {
		// Read in tCmd
		tCmd := strings.TrimSpace(fileToString(testFile))
		// Transform cliReplace to true app path
		tCmd = strings.Replace(tCmd, cliReplace, cliRel, -1)
		// Execute tCmd in shell and save stdout, stderr, exit
		cmd := exec.Command(shell, "-c", tCmd)
		var stdout, stderr strings.Builder
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
		cmd.Dir = testPath
		err := cmd.Run()

		var exitCode string
		if err != nil {
			if exitError, ok := err.(*exec.ExitError); ok {
				exitCode = strconv.Itoa(exitError.ExitCode())
			} else {
				panic("Error getting exit code")
			}
		} else {
			exitCode = "0"
		}
		checkExpected(t, testPath, "tExit", exitCode)

		checkExpected(t, testPath, "tStdout", stdout.String())

		checkExpected(t, testPath, "tStderr", stderr.String())

		// Compare any t*.[check|checkRegex] files against t*.result files
		r := regexp.MustCompile(`^t.*\.(check|checkRegex)$`)
		err = filepath.Walk(testPath, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			// Is it a check file?
			if r.MatchString(filepath.Base(path)) {
				fDir := filepath.Dir(path)
				fBase := filepath.Base(path)
				fNoext := strings.TrimSuffix(fBase, filepath.Ext(fBase))
				tResult := fileToString(fDir + "/" + fNoext + resultExt)
				checkExpected(t, fDir, fNoext+checkExt, tResult)
			}
			return nil
		})
		if err != nil {
			panic("Unable to walk directory tree for t*.[check|checkRegex] files")
		}
	})
	return nil
}

// Check the actual against the expected output.
func checkExpected(t *testing.T, testPath, fName, actual string) {
	fNameR := fName + "Regex"
	fPath := testPath + "/" + fName
	fPathR := testPath + "/" + fNameR
	if _, err := os.Stat(fPath); err == nil {
		// Direct string comparison
		fCheck := fileToString(fPath)
		if fCheck != actual {
			t.Errorf(fName+":\n expected %q\n received %q", fCheck, actual)
		}
	} else if _, err := os.Stat(fPathR); err == nil {
		// Regex comparison
		fCheck := fileToString(fPathR)
		r := regexp.MustCompile(fCheck)
		if !r.MatchString(actual) {
			t.Errorf(fNameR+":\n regex %q\n did not match %q", fCheck, actual)
		}
	} else {
		panic("Test file " + testPath + "[" + fName + "|" + fNameR + "] not found.")
	}
}

// Read the specified file and return it as a string.
func fileToString(fPath string) string {
	fByte, err := ioutil.ReadFile(fPath)
	if err != nil {
		panic("Unable to read " + fPath)
	}
	return string(fByte)
}