pkg/eval/evaltest: Remove the old testing framework.

This commit is contained in:
Qi Xiao 2024-01-30 20:53:09 +00:00
parent 27495be1b9
commit d4f11db983
5 changed files with 9 additions and 760 deletions

View File

@ -1,136 +0,0 @@
package evaltest
import (
"fmt"
"reflect"
"src.elv.sh/pkg/eval"
"src.elv.sh/pkg/parse"
)
type errorMatcher interface{ matchError(error) bool }
// An errorMatcher for compilation errors.
type compilationError struct {
msgs []string
}
func (e compilationError) Error() string {
return fmt.Sprintf("compilation errors with messages: %v", e.msgs)
}
func (e compilationError) matchError(e2 error) bool {
errs := eval.UnpackCompilationErrors(e2)
if len(e.msgs) != len(errs) {
return false
}
for i, msg := range e.msgs {
if msg != errs[i].Message {
return false
}
}
return true
}
// An errorMatcher for exceptions.
type exc struct {
reason error
stacks []string
}
func (e exc) Error() string {
if len(e.stacks) == 0 {
return fmt.Sprintf("exception with reason %v", e.reason)
}
return fmt.Sprintf("exception with reason %v and stacks %v", e.reason, e.stacks)
}
func (e exc) matchError(e2 error) bool {
if e2, ok := e2.(eval.Exception); ok {
return matchErr(e.reason, e2.Reason()) &&
(len(e.stacks) == 0 ||
reflect.DeepEqual(e.stacks, getStackTexts(e2.StackTrace())))
}
return false
}
func getStackTexts(tb *eval.StackTrace) []string {
texts := []string{}
for tb != nil {
ctx := tb.Head
texts = append(texts, ctx.Body)
tb = tb.Next
}
return texts
}
// AnyParseError is an error that can be passed to the Case.Throws to match any
// parse error.
var AnyParseError anyParseError
type anyParseError struct{}
func (anyParseError) Error() string { return "any parse error" }
func (anyParseError) matchError(e error) bool { return parse.UnpackErrors(e) != nil }
// ErrorWithType returns an error that can be passed to the Case.Throws to match
// any error with the same type as the argument.
func ErrorWithType(v error) error { return errWithType{v} }
// An errorMatcher for any error with the given type.
type errWithType struct{ v error }
func (e errWithType) Error() string { return fmt.Sprintf("error with type %T", e.v) }
func (e errWithType) matchError(e2 error) bool {
return reflect.TypeOf(e.v) == reflect.TypeOf(e2)
}
// ErrorWithMessage returns an error that can be passed to Case.Throws to match
// any error with the given message.
func ErrorWithMessage(msg string) error { return errWithMessage{msg} }
// An errorMatcher for any error with the given message.
type errWithMessage struct{ msg string }
func (e errWithMessage) Error() string { return "error with message " + e.msg }
func (e errWithMessage) matchError(e2 error) bool {
return e2 != nil && e.msg == e2.Error()
}
// CmdExit returns an error that can be passed to Case.Throws to match an
// eval.ExternalCmdExit ignoring the Pid field.
func CmdExit(v eval.ExternalCmdExit) error { return errCmdExit{v} }
// An errorMatcher for an ExternalCmdExit error that ignores the `Pid` member.
// We only match the command name and exit status because at run time we
// cannot know the correct value for `Pid`.
type errCmdExit struct{ v eval.ExternalCmdExit }
func (e errCmdExit) Error() string {
return e.v.Error()
}
func (e errCmdExit) matchError(gotErr error) bool {
if gotErr == nil {
return false
}
ge := gotErr.(eval.ExternalCmdExit)
return e.v.CmdName == ge.CmdName && e.v.WaitStatus == ge.WaitStatus
}
type errOneOf struct{ errs []error }
func OneOfErrors(errs ...error) error { return errOneOf{errs} }
func (e errOneOf) Error() string { return fmt.Sprint("one of", e.errs) }
func (e errOneOf) matchError(gotError error) bool {
for _, want := range e.errs {
if matchErr(want, gotError) {
return true
}
}
return false
}

View File

@ -1,306 +0,0 @@
// Package evaltest provides a framework for testing Elvish script.
//
// The entry point for the framework is the Test function, which accepts a
// *testing.T and any number of test cases.
//
// Test cases are constructed using the That function, followed by method calls
// that add additional information to it.
//
// Example:
//
// Test(t,
// That("put x").Puts("x"),
// That("echo x").Prints("x\n"))
//
// If some setup is needed, use the TestWithSetup function instead.
package evaltest
import (
"bytes"
"fmt"
"reflect"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"src.elv.sh/pkg/diag"
"src.elv.sh/pkg/eval"
"src.elv.sh/pkg/eval/vals"
"src.elv.sh/pkg/must"
"src.elv.sh/pkg/parse"
)
// Case is a test case that can be used in Test.
type Case struct {
codes []string
setup func(ev *eval.Evaler)
verify func(t *testing.T)
want result
}
type result struct {
ValueOut []any
BytesOut []byte
StderrOut []byte
CompilationError error
Exception error
}
// That returns a new Case with the specified source code. Multiple arguments
// are joined with newlines. To specify multiple pieces of code that are
// executed separately, use the Then method to append code pieces.
//
// When combined with subsequent method calls, a test case reads like English.
// For example, a test for the fact that "put x" puts "x" reads:
//
// That("put x").Puts("x")
func That(lines ...string) Case {
return Case{codes: []string{strings.Join(lines, "\n")}}
}
// Then returns a new Case that executes the given code in addition. Multiple
// arguments are joined with newlines.
func (c Case) Then(lines ...string) Case {
c.codes = append(c.codes, strings.Join(lines, "\n"))
return c
}
// Then returns a new Case with the given setup function executed on the Evaler
// before the code is executed.
func (c Case) WithSetup(f func(*eval.Evaler)) Case {
c.setup = f
return c
}
// DoesNothing returns t unchanged. It is useful to mark tests that don't have
// any side effects, for example:
//
// That("nop").DoesNothing()
func (c Case) DoesNothing() Case {
return c
}
// Puts returns an altered Case that runs an additional verification function.
func (c Case) Passes(f func(t *testing.T)) Case {
c.verify = f
return c
}
// Puts returns an altered Case that requires the source code to produce the
// specified values in the value channel when evaluated.
func (c Case) Puts(vs ...any) Case {
c.want.ValueOut = vs
return c
}
// Prints returns an altered Case that requires the source code to produce the
// specified output in the byte pipe when evaluated.
func (c Case) Prints(s string) Case {
c.want.BytesOut = []byte(s)
return c
}
// PrintsStderrWith returns an altered Case that requires the stderr output to
// contain the given text.
func (c Case) PrintsStderrWith(s string) Case {
c.want.StderrOut = []byte(s)
return c
}
// Throws returns an altered Case that requires the source code to throw an
// exception with the given reason. The reason supports special matcher values
// constructed by functions like ErrorWithMessage.
//
// If at least one stacktrace string is given, the exception must also have a
// stacktrace matching the given source fragments, frame by frame (innermost
// frame first). If no stacktrace string is given, the stack trace of the
// exception is not checked.
func (c Case) Throws(reason error, stacks ...string) Case {
c.want.Exception = exc{reason, stacks}
return c
}
// DoesNotCompile returns an altered Case that requires the source code to fail
// compilation with the given error messages.
func (c Case) DoesNotCompile(msgs ...string) Case {
c.want.CompilationError = compilationError{msgs}
return c
}
// Test is a shorthand for [TestWithSetup] when no setup is needed.
func Test(t *testing.T, tests ...Case) {
t.Helper()
testWithSetup(t, func(*testing.T, *eval.Evaler) {}, tests...)
}
// TestWithEvalerSetup is a shorthand for [TestWithSetup] when the setup only
// needs to manipulate [eval.Evaler].
func TestWithEvalerSetup(t *testing.T, setup func(*eval.Evaler), tests ...Case) {
t.Helper()
testWithSetup(t, func(_ *testing.T, ev *eval.Evaler) { setup(ev) }, tests...)
}
// TestWithSetup runs test cases.
//
// Each test case is run as a subtest with a newly created Evaler. The setup
// function is called with the [testing.T] and the [eval.Evaler] for the subset
// before code evaluation.
func TestWithSetup(t *testing.T, setup func(*testing.T, *eval.Evaler), tests ...Case) {
t.Helper()
testWithSetup(t, setup, tests...)
}
func testWithSetup(t *testing.T, setup func(*testing.T, *eval.Evaler), tests ...Case) {
t.Helper()
ew := newElvtsWriter()
defer ew.Close()
for _, tc := range tests {
t.Run(strings.Join(tc.codes, "\n"), func(t *testing.T) {
t.Helper()
ev := eval.NewEvaler()
setup(t, ev)
if tc.setup != nil {
tc.setup(ev)
}
r := evalAndCollect(t, ev, tc.codes)
ew.writeCase(tc, r)
if tc.verify != nil {
tc.verify(t)
}
if !matchOut(tc.want.ValueOut, r.ValueOut) {
t.Errorf("got value out (-want +got):\n%s",
cmp.Diff(reprValues(tc.want.ValueOut), reprValues(r.ValueOut)))
}
if !bytes.Equal(tc.want.BytesOut, r.BytesOut) {
t.Errorf("got bytes out (-want +got):\n%s",
cmp.Diff(string(tc.want.BytesOut), string(r.BytesOut)))
}
if tc.want.StderrOut == nil {
if len(r.StderrOut) > 0 {
t.Errorf("got stderr out %q, want empty", r.StderrOut)
}
} else {
if !bytes.Contains(r.StderrOut, tc.want.StderrOut) {
t.Errorf("got stderr out %q, want output containing %q",
r.StderrOut, tc.want.StderrOut)
}
}
if !matchErr(tc.want.CompilationError, r.CompilationError) {
t.Errorf("got compilation error:\n%v\nwant %v",
show(r.CompilationError), tc.want.CompilationError)
}
if !matchErr(tc.want.Exception, r.Exception) {
t.Errorf("unexpected exception")
if exc, ok := r.Exception.(eval.Exception); ok {
// For an eval.Exception report the type of the underlying error.
t.Logf("got: %T: %v", exc.Reason(), exc)
t.Logf("stack trace: %#v", getStackTexts(exc.StackTrace()))
} else {
t.Logf("got: %T: %v", r.Exception, r.Exception)
}
t.Errorf("want: %v", tc.want.Exception)
}
})
}
}
func reprValues(xs []any) []string {
rs := make([]string, len(xs))
for i, x := range xs {
rs[i] = vals.Repr(x, 0)
}
return rs
}
// Use returns a function simulates "use" on an Evaler. Arguments must come in
// (string, eval.Nser) pairs.
func Use(args ...any) func(*eval.Evaler) {
if len(args)%2 != 0 {
panic("odd number of arguments")
}
ns := eval.BuildNs()
for i := 0; i < len(args); i += 2 {
ns.AddNs(args[i].(string), args[i+1].(eval.Nser))
}
return func(ev *eval.Evaler) {
ev.ExtendGlobal(ns)
}
}
func evalAndCollect(t *testing.T, ev *eval.Evaler, texts []string) result {
var r result
port1, collect1 := must.OK2(eval.CapturePort())
port2, collect2 := must.OK2(eval.CapturePort())
ports := []*eval.Port{eval.DummyInputPort, port1, port2}
for _, text := range texts {
ctx, done := eval.ListenInterrupts()
err := ev.Eval(parse.Source{Name: "[tty]", Code: text},
eval.EvalCfg{Ports: ports, Interrupts: ctx})
done()
if parse.UnpackErrors(err) != nil {
t.Fatalf("Parse(%q) error: %s", text, err)
} else if eval.UnpackCompilationErrors(err) != nil {
// NOTE: If multiple code pieces have compilation errors, only the
// last one compilation error is saved.
r.CompilationError = err
} else if err != nil {
// NOTE: If multiple code pieces throw exceptions, only the last one
// is saved.
r.Exception = err
}
}
r.ValueOut, r.BytesOut = collect1()
_, r.StderrOut = collect2()
return r
}
func matchOut(want, got []any) bool {
if len(got) != len(want) {
return false
}
for i := range got {
if !match(got[i], want[i]) {
return false
}
}
return true
}
func match(got, want any) bool {
if matcher, ok := want.(ValueMatcher); ok {
return matcher.matchValue(got)
}
// Special-case float64 to handle NaNs and infinities.
if got, ok := got.(float64); ok {
if want, ok := want.(float64); ok {
return matchFloat64(got, want, 0)
}
}
return vals.Equal(got, want)
}
func matchErr(want, got error) bool {
if want == nil {
return got == nil
}
if matcher, ok := want.(errorMatcher); ok {
return matcher.matchError(got)
}
return reflect.DeepEqual(want, got)
}
func show(v any) string {
if s, ok := v.(diag.Shower); ok {
return s.Show("")
}
return fmt.Sprint(v)
}

View File

@ -1,3 +1,4 @@
// Package evaltest supports testing the Elvish interpreter and libraries.
package evaltest
import (
@ -230,3 +231,11 @@ func stripSGR(bs []byte) []byte { return sgrPattern.ReplaceAllLiteral(bs, n
func stripSGRString(s string) string { return sgrPattern.ReplaceAllLiteralString(s, "") }
func normalizeLineEnding(bs []byte) []byte { return bytes.ReplaceAll(bs, []byte("\r\n"), []byte("\n")) }
// Use returns a function that simulates "use" on an Evaler and can be used as a
// setup function for [TestTranscriptsInFS].
func Use(name string, ns eval.Nser) func(*eval.Evaler) {
return func(ev *eval.Evaler) {
ev.ExtendGlobal(eval.BuildNs().AddNs(name, ns))
}
}

View File

@ -1,102 +0,0 @@
package evaltest
import (
"math"
"math/big"
"regexp"
"src.elv.sh/pkg/eval/vals"
)
// ValueMatcher is a value that can be passed to [Case.Puts] and has its own
// matching semantics.
type ValueMatcher interface{ matchValue(any) bool }
// Anything matches anything. It is useful when the value contains information
// that is useful when the test fails.
var Anything ValueMatcher = anything{}
type anything struct{}
func (anything) matchValue(any) bool { return true }
// AnyInteger matches any integer.
var AnyInteger ValueMatcher = anyInteger{}
type anyInteger struct{}
func (anyInteger) matchValue(x any) bool {
switch x.(type) {
case int, *big.Int:
return true
default:
return false
}
}
// ApproximatelyThreshold defines the threshold for matching float64 values when
// using [Approximately].
const ApproximatelyThreshold = 1e-15
// Approximately matches a float64 within the threshold defined by
// [ApproximatelyThreshold].
func Approximately(f float64) ValueMatcher { return approximately{f} }
type approximately struct{ value float64 }
func (a approximately) matchValue(value any) bool {
if value, ok := value.(float64); ok {
return matchFloat64(a.value, value, ApproximatelyThreshold)
}
return false
}
func matchFloat64(a, b, threshold float64) bool {
if math.IsNaN(a) && math.IsNaN(b) {
return true
}
if math.IsInf(a, 0) && math.IsInf(b, 0) &&
math.Signbit(a) == math.Signbit(b) {
return true
}
return math.Abs(a-b) <= threshold
}
// [StringMatching] matches any string matching a regexp pattern. If the pattern
// is not a valid regexp, the function panics.
func StringMatching(p string) ValueMatcher { return stringMatching{regexp.MustCompile(p)} }
type stringMatching struct{ pattern *regexp.Regexp }
func (s stringMatching) matchValue(value any) bool {
if value, ok := value.(string); ok {
return s.pattern.MatchString(value)
}
return false
}
// MapContaining matches any map that contains all the key-value pairs in the
// given map. The values in the argument itself can also be [ValueMatcher]s.
func MapContaining(m vals.Map) ValueMatcher { return mapContaining{m} }
// MapContainingPairs is a shorthand for MapContaining(vals.MakeMap(a...)).
func MapContainingPairs(a ...any) ValueMatcher { return MapContaining(vals.MakeMap(a...)) }
type mapContaining struct{ m vals.Map }
func (m mapContaining) matchValue(value any) bool {
if gotMap, ok := value.(vals.Map); ok {
for it := m.m.Iterator(); it.HasElem(); it.Next() {
k, wantValue := it.Elem()
if gotValue, ok := gotMap.Index(k); !ok || !match(gotValue, wantValue) {
return false
}
}
return true
}
return false
}
func (m mapContaining) Repr(indent int) string {
return vals.Repr(m.m, indent)
}

View File

@ -1,216 +0,0 @@
package evaltest
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"src.elv.sh/pkg/diag"
"src.elv.sh/pkg/eval/vals"
)
const generatedElvTsBanner = "// Generated, can be overwritten"
var elvtsFileStatus = map[string]int{}
// Writes a .elvts file from invocation of Test.
//
// Each test case is its own transcript session. The function name is used as
// h1, and the comment above a test case in the Go source code is used as h2,
// with a fallback title if there are no comments.
type elvtsWriter struct {
w io.Writer
titleForCode map[string]string
}
func newElvtsWriter() elvtsWriter {
// Actual test site -> Test/TestWithSetup/TestWithEvalerSetup ->
// testWithSetup -> this function
pc, filename, line, _ := runtime.Caller(3)
elvtsName := filename[:len(filename)-len(filepath.Ext(filename))] + ".elvts"
var w io.Writer
w, elvtsFileStatus[elvtsName] = getElvts(elvtsName, elvtsFileStatus[elvtsName])
funcname := runtime.FuncForPC(pc).Name()
if i := strings.LastIndexByte(funcname, '.'); i != -1 {
funcname = funcname[i+1:]
}
fmt.Fprintf(w, "# %s #\n", funcname)
return elvtsWriter{w, parseTestTitles(filename, line)}
}
func (ew elvtsWriter) writeCase(c Case, r result) {
// Add empty line before title
fmt.Fprintln(ew.w)
title := "?"
// The parseTestTitles heuristics assumes a single piece of code (which is
// the vast majority anyway).
if len(c.codes) == 1 && ew.titleForCode[c.codes[0]] != "" {
title = ew.titleForCode[c.codes[0]]
}
fmt.Fprintf(ew.w, "## %s ##\n", title)
fmt.Fprintf(ew.w, "~> %s\n", c.codes[0])
for _, line := range c.codes[1:] {
fmt.Fprintf(ew.w, " %s\n", line)
}
fmt.Fprint(ew.w, stringifyResult(r))
}
func (ew elvtsWriter) Close() error {
if closer, ok := ew.w.(io.Closer); ok {
return closer.Close()
}
return nil
}
// 0 for unknown
// 1 for already generated
// 2 for don't overwrite
func getElvts(name string, status int) (io.Writer, int) {
switch status {
case 0:
if mayOverwriteElvts(name) {
file, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if err != nil {
panic(err)
}
fmt.Fprintln(file, generatedElvTsBanner)
fmt.Fprintln(file, "//only-on ignore")
return file, 1
} else {
return io.Discard, 2
}
case 1:
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
panic(err)
}
fmt.Fprintln(file)
return file, 1
case 2:
return io.Discard, 2
}
panic("bad status")
}
func mayOverwriteElvts(name string) bool {
file, err := os.Open(name)
if err != nil {
if os.IsNotExist(err) {
return true
}
panic(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Scan()
return scanner.Text() == generatedElvTsBanner
}
// Parses a block of code that looks like:
//
// Test(t,
// // A title
// That("code 1")...,
// That("code 2")...,
// )
//
// and extracts titles for the test code:
//
// map[string]string{
// "code 1": "A title",
// }
func parseTestTitles(filename string, fromLine int) map[string]string {
file, err := os.Open(filename)
if err != nil {
panic(err)
}
m := make(map[string]string)
reader := bufio.NewScanner(file)
lineno := 0
for reader.Scan() {
lineno++
if lineno < fromLine {
continue
}
line := strings.TrimSpace(reader.Text())
if line == ")" {
break
}
if title, ok := parseComment(line); ok {
// Titled clause; collect all comment lines.
for reader.Scan() {
line := strings.TrimSpace(reader.Text())
if moreTitle, ok := parseComment(line); ok {
title = title + " " + moreTitle
} else if code, ok := parseThatCode(line); ok {
if oldTitle, exists := m[code]; exists {
fmt.Printf("%s:%d: title for %q: %q (already has %q)\n",
filepath.Base(filename), lineno, code, title, oldTitle)
} else {
m[code] = title
}
break
} else {
break
}
}
}
}
if reader.Err() != nil {
panic(reader.Err())
}
return m
}
// "// comment" -> "comment", true
// Everything else -> ?, false
func parseComment(line string) (string, bool) {
return strings.CutPrefix(line, "// ")
}
var (
doubleQuoted = regexp.MustCompile(`^"(?:[^\\"]|\\")*"`)
backQuoted = regexp.MustCompile("^`[^`]*`")
)
// `That("code")...` -> "code", true
// "That(`code`)..." -> "code", true
// Everything else -> "", false
func parseThatCode(line string) (string, bool) {
// TODO
if rest, ok := strings.CutPrefix(line, "That("); ok {
if dq := doubleQuoted.FindString(rest); dq != "" {
return strings.ReplaceAll(dq[1:len(dq)-1], `\"`, `"`), true
} else if bq := backQuoted.FindString(rest); bq != "" {
return bq[1 : len(bq)-1], true
}
}
return "", false
}
func stringifyResult(r result) string {
var sb strings.Builder
for _, value := range r.ValueOut {
sb.WriteString(valuePrefix + vals.ReprPlain(value) + "\n")
}
sb.Write(stripSGR(r.BytesOut))
sb.Write(stripSGR(r.StderrOut))
if r.CompilationError != nil {
sb.WriteString(stripSGRString(r.CompilationError.(diag.Shower).Show("")) + "\n")
}
if r.Exception != nil {
sb.WriteString(stripSGRString(r.Exception.(diag.Shower).Show("")) + "\n")
}
return sb.String()
}