// Package evaltest supports testing the Elvish interpreter and libraries. package evaltest import ( "bytes" "fmt" "go/build/constraint" "io/fs" "regexp" "runtime" "strings" "testing" "src.elv.sh/pkg/diag" "src.elv.sh/pkg/diff" "src.elv.sh/pkg/eval" "src.elv.sh/pkg/eval/vals" "src.elv.sh/pkg/mods" "src.elv.sh/pkg/must" "src.elv.sh/pkg/parse" "src.elv.sh/pkg/testutil" "src.elv.sh/pkg/transcript" ) // TestTranscriptsInFS extracts all Elvish transcript sessions from .elv and // .elvts files in fsys, and runs each of them as a test. See // [src.elv.sh/pkg/transcript] for how transcript sessions are discovered. // // Typical use of this function looks like this: // // import ( // "embed" // "src.elv.sh/pkg/eval/evaltest" // ) // // //go:embed *.elv *.elvts // var transcripts embed.FS // // func TestTranscripts(t *testing.T) { // evaltest.TestTranscriptsInFS(t, transcripts) // } // // The function accepts variadic arguments in (name, f) pairs, where name must // not contain any spaces. Each pair defines a setup function that may be // referred to in the transcripts with the directive "//name". // // The setup function f may take a *testing.T, *eval.Evaler and a string // argument. All of them are optional but must appear in that order. If it takes // a string argument, the directive can be followed by an argument after a space // ("//name argument"), and that argument is passed to f. The argument itself // may contain spaces. // // The following setup functions are predefined: // // - in-temp-dir: Run inside a temporary directory. // // - set-env $name $value: Run with the environment variable $name set to // $value. // // - unset-env $name: Run with the environment variable $name unset. // // - eval $code: Evaluate the argument as Elvish code. // // - only-on $cond: Evaluate $cond like a //go:build constraint and only // run the test if the constraint is satisfied. // // The full build constraint syntax is supported, but only literal GOARCH // and GOOS values and "unix" are recognized as tags; other tags are always // false. // // Since directives in a higher level propagated to all its descendants, this // mechanism can be used to specify setup functions that apply to an entire // .elvts file (or an entire elvish-transcript code block in a .elv file) or an // entire section: // // //global-setup // // # h1 # // //h1-setup // // ## h2 ## // //h2-setup // // // All of top-setup, h1-setup and h2-setup are run for this session, in that // // order. // // ~> echo foo // foo func TestTranscriptsInFS(t *testing.T, fsys fs.FS, setupPairs ...any) { sessions, err := transcript.ParseSessionsInFS(fsys) if err != nil { t.Fatalf("parse transcript sessions: %v", err) } testTranscripts(t, sessions, setupPairs) } func testTranscripts(t *testing.T, sessions []transcript.Session, setupPairs []any) { setupMap, argSetupMap := buildSetupMaps(setupPairs) for _, session := range sessions { t.Run(session.Name, func(t *testing.T) { ev := eval.NewEvaler() mods.AddTo(ev) for _, directive := range session.Directives { name, arg, _ := strings.Cut(directive, " ") if f, ok := setupMap[name]; ok { if arg != "" { t.Fatalf("setup function %s doesn't support arguments", name) } f(t, ev) } else if f, ok := argSetupMap[name]; ok { f(t, ev, arg) } else { t.Fatalf("unknown setup function: %s", name) } } for _, interaction := range session.Interactions { want := interaction.Output got := evalAndCollectOutput(t, ev, interaction.Code) if want != got { t.Errorf("\n%s\n-want +got:\n%s", interaction.PromptAndCode(), diff.DiffNoHeader(want, got)) } } }) } } func buildSetupMaps(setupPairs []any) (map[string]func(*testing.T, *eval.Evaler), map[string]func(*testing.T, *eval.Evaler, string)) { if len(setupPairs)%2 != 0 { panic(fmt.Sprintf("variadic arguments must come in pairs, got %d", len(setupPairs))) } setupMap := map[string]func(*testing.T, *eval.Evaler){ "in-temp-dir": func(t *testing.T, ev *eval.Evaler) { testutil.InTempDir(t) }, } argSetupMap := map[string]func(*testing.T, *eval.Evaler, string){ "set-env": func(t *testing.T, ev *eval.Evaler, arg string) { name, value, _ := strings.Cut(arg, " ") testutil.Setenv(t, name, value) }, "unset-env": func(t *testing.T, ev *eval.Evaler, name string) { testutil.Unsetenv(t, name) }, "eval": func(t *testing.T, ev *eval.Evaler, code string) { err := ev.Eval( parse.Source{Name: "[setup]", Code: code}, eval.EvalCfg{Ports: eval.DummyPorts}) if err != nil { t.Fatalf("setup failed: %v\n", err) } }, "only-on": func(t *testing.T, _ *eval.Evaler, arg string) { expr, err := constraint.Parse("//go:build " + arg) if err != nil { t.Fatal(err) } if !expr.Eval(func(tag string) bool { if tag == "unix" { return isUNIX } return tag == runtime.GOOS || tag == runtime.GOARCH }) { t.Skipf("constraint not satisfied: %s", arg) } }, } for i := 0; i < len(setupPairs); i += 2 { name := setupPairs[i].(string) if setupMap[name] != nil || argSetupMap[name] != nil { panic(fmt.Sprintf("there's already a setup functions named %s", name)) } switch f := setupPairs[i+1].(type) { case func(): setupMap[name] = func(_ *testing.T, _ *eval.Evaler) { f() } case func(*testing.T): setupMap[name] = func(t *testing.T, ev *eval.Evaler) { f(t) } case func(*eval.Evaler): setupMap[name] = func(t *testing.T, ev *eval.Evaler) { f(ev) } case func(*testing.T, *eval.Evaler): setupMap[name] = f case func(string): argSetupMap[name] = func(_ *testing.T, _ *eval.Evaler, s string) { f(s) } case func(*testing.T, string): argSetupMap[name] = func(t *testing.T, _ *eval.Evaler, s string) { f(t, s) } case func(*eval.Evaler, string): argSetupMap[name] = func(_ *testing.T, ev *eval.Evaler, s string) { f(ev, s) } case func(*testing.T, *eval.Evaler, string): argSetupMap[name] = f default: panic(fmt.Sprintf("unsupported setup function type: %T", f)) } } return setupMap, argSetupMap } var valuePrefix = "▶ " func evalAndCollectOutput(t *testing.T, ev *eval.Evaler, code string) string { port1, collect1 := must.OK2(eval.CapturePort()) port2, collect2 := must.OK2(eval.CapturePort()) ports := []*eval.Port{eval.DummyInputPort, port1, port2} ctx, done := eval.ListenInterrupts() err := ev.Eval( parse.Source{Name: "[tty]", Code: code}, eval.EvalCfg{Ports: ports, Interrupts: ctx}) done() values, stdout := collect1() _, stderr := collect2() var sb strings.Builder for _, value := range values { sb.WriteString(valuePrefix + vals.ReprPlain(value) + "\n") } sb.Write(normalizeLineEnding(stripSGR(stdout))) sb.Write(normalizeLineEnding(stripSGR(stderr))) if err != nil { if shower, ok := err.(diag.Shower); ok { sb.WriteString(stripSGRString(shower.Show(""))) } else { sb.WriteString(err.Error()) } sb.WriteByte('\n') } return sb.String() } var sgrPattern = regexp.MustCompile("\033\\[[0-9;]*m") func stripSGR(bs []byte) []byte { return sgrPattern.ReplaceAllLiteral(bs, nil) } func stripSGRString(s string) string { return sgrPattern.ReplaceAllLiteralString(s, "") } func normalizeLineEnding(bs []byte) []byte { return bytes.ReplaceAll(bs, []byte("\r\n"), []byte("\n")) }