website/cmd/md2html: Reduce dependency on Elvish packages.

This command used to depend on pkg/mods/doc to access the embedded .elv, which
in turn depends on all the packages that implement builtin modules. The latter
set of packages depends on almost all the Elvish packages transitively. As a
result, almost any change in any Elvish package will trigger a rebuild of this
command and the whole website.

This commit minimizes the dependency on Elvish packages by having it read the
.elv files during runtime instead (enabled by
9112eb1ab2).

Additionally:

- Move HighlightCodeBlock, needed by website/cmd/md2html, from pkg/mods/doc
  to pkg/elvdoc. Moving it is necessary to completely remove the dependency of
  website/cmd/md2html on pkg/mods/doc.

- Remove the dependency of pkg/edit/highlight on pkg/eval. It only uses
  eval.UnpackCompilationErrors; move this work to the supplied Check function.
  This removes the transitive dependency of website/cmd/md2html on pkg/eval.

- Augment website/tools/md-deps to recognize @module lines and add dependency on
  the corresponding .elv files.

pkg/edit/highlight.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"src.elv.sh/pkg/cli"
+	"src.elv.sh/pkg/diag"
 	"src.elv.sh/pkg/edit/highlight"
 	"src.elv.sh/pkg/eval"
 	"src.elv.sh/pkg/fsutil"
@@ -15,11 +16,11 @@ import (
 
 func initHighlighter(appSpec *cli.AppSpec, ed *Editor, ev *eval.Evaler, nb eval.NsBuilder) {
 	hl := highlight.NewHighlighter(highlight.Config{
-		Check: func(t parse.Tree) (string, error) {
+		Check: func(t parse.Tree) (string, []*diag.Error) {
 			autofixes, err := ev.CheckTree(t, nil)
 			autofix := strings.Join(autofixes, "; ")
 			ed.autofix.Store(autofix)
-			return autofix, err
+			return autofix, eval.UnpackCompilationErrors(err)
 		},
 		HasCommand: func(cmd string) bool { return hasCommand(ev, cmd) },
 		AutofixTip: func(autofix string) ui.Text {

pkg/edit/highlight/highlight.go

@@ -5,14 +5,13 @@ import (
 	"time"
 
 	"src.elv.sh/pkg/diag"
-	"src.elv.sh/pkg/eval"
 	"src.elv.sh/pkg/parse"
 	"src.elv.sh/pkg/ui"
 )
 
 // Config keeps configuration for highlighting code.
 type Config struct {
-	Check      func(n parse.Tree) (string, error)
+	Check      func(n parse.Tree) (string, []*diag.Error)
 	HasCommand func(name string) bool
 	AutofixTip func(autofix string) ui.Text
 }
@@ -46,8 +45,8 @@ func highlight(code string, cfg Config, lateCb func(ui.Text)) (ui.Text, []ui.Tex
 	}
 
 	if cfg.Check != nil {
-		autofix, errCheck := cfg.Check(tree)
-		for _, err := range eval.UnpackCompilationErrors(errCheck) {
+		autofix, diagErrors := cfg.Check(tree)
+		for _, err := range diagErrors {
 			addDiagError(err)
 		}
 		if autofix != "" && cfg.AutofixTip != nil {

pkg/edit/highlight/highlighter_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"src.elv.sh/pkg/diag"
 	"src.elv.sh/pkg/eval"
 	"src.elv.sh/pkg/parse"
 	"src.elv.sh/pkg/testutil"
@@ -83,9 +84,9 @@ func TestHighlighter_AutofixesAndCheckErrors(t *testing.T) {
 	ev := eval.NewEvaler()
 	ev.AddModule("mod1", &eval.Ns{})
 	hl := NewHighlighter(Config{
-		Check: func(t parse.Tree) (string, error) {
+		Check: func(t parse.Tree) (string, []*diag.Error) {
 			autofixes, err := ev.CheckTree(t, nil)
-			return strings.Join(autofixes, "; "), err
+			return strings.Join(autofixes, "; "), eval.UnpackCompilationErrors(err)
 		},
 		AutofixTip: func(s string) ui.Text { return ui.T("autofix: " + s) },
 	})

pkg/elvdoc/highlight.go

@@ -1,4 +1,4 @@
-package doc
+package elvdoc
 
 import (
 	"regexp"

pkg/elvdoc/highlight_test.go

@@ -1,13 +1,15 @@
-package doc_test
+package elvdoc_test
 
 import (
 	"reflect"
 	"testing"
 
-	"src.elv.sh/pkg/mods/doc"
+	"src.elv.sh/pkg/elvdoc"
+	"src.elv.sh/pkg/testutil"
 	"src.elv.sh/pkg/ui"
 )
 
+var Dedent = testutil.Dedent
 var stylesheet = ui.RuneStylesheet{
 	'v': ui.FgGreen, '$': ui.FgMagenta,
 }
@@ -68,7 +70,7 @@ var highlightCodeBlockTests = []struct {
 func TestHighlightCodeBlock(t *testing.T) {
 	for _, tc := range highlightCodeBlockTests {
 		t.Run(tc.name, func(t *testing.T) {
-			got := doc.HighlightCodeBlock(tc.info, tc.code)
+			got := elvdoc.HighlightCodeBlock(tc.info, tc.code)
 			if !reflect.DeepEqual(got, tc.want) {
 				t.Errorf("got %s, want %s", got, tc.want)
 			}

pkg/mods/doc/doc.go

@@ -43,7 +43,7 @@ func show(fm *eval.Frame, opts showOptions, fqname string) error {
 	}
 	codec := &md.TTYCodec{
 		Width:              width,
-		HighlightCodeBlock: HighlightCodeBlock,
+		HighlightCodeBlock: elvdoc.HighlightCodeBlock,
 		ConvertRelativeLink: func(dest string) string {
 			// TTYCodec does not show destinations of relative links by default.
 			// Special-case links to language.html as they are quite common in
@@ -66,7 +66,7 @@ func show(fm *eval.Frame, opts showOptions, fqname string) error {
 }
 
 func find(fm *eval.Frame, qs ...string) {
-	for ns, docs := range Docs() {
+	for ns, docs := range docs() {
 		findIn := func(name, markdown string) {
 			if bs, ok := match(markdown, qs); ok {
 				out := fm.ByteOutput()
@@ -97,7 +97,7 @@ func Source(fqname string) (string, error) {
 	} else if first == "builtin:" {
 		first = ""
 	}
-	docs, ok := Docs()[first]
+	docs, ok := docs()[first]
 	if !ok {
 		return "", fmt.Errorf("no doc for %s", parse.Quote(fqname))
 	}
@@ -118,7 +118,7 @@ func Source(fqname string) (string, error) {
 
 func symbols(fm *eval.Frame) error {
 	var names []string
-	for ns, docs := range Docs() {
+	for ns, docs := range docs() {
 		for _, fn := range docs.Fns {
 			names = append(names, ns+fn.Name)
 		}
@@ -138,17 +138,19 @@ func symbols(fm *eval.Frame) error {
 
 var (
 	docsOnce sync.Once
-	docs     map[string]elvdoc.Docs
+	docsMap  map[string]elvdoc.Docs
 	// May be overridden in tests.
 	elvFiles fs.FS = pkg.ElvFiles
 )
 
-// Docs returns a map from namespace prefixes (like "doc:", or "" for the
-// builtin module) to extracted elvdocs.
-func Docs() map[string]elvdoc.Docs {
+// Returns a map from namespace prefixes (like "doc:", or "" for the builtin
+// module) to extracted elvdocs.
+//
+// TODO: Simplify this using [sync.OnceValue] once Elvish requires Go 1.21.
+func docs() map[string]elvdoc.Docs {
 	docsOnce.Do(func() {
 		// We don't expect any errors from reading an [embed.FS].
-		docs, _ = elvdoc.ExtractAllFromFS(elvFiles)
+		docsMap, _ = elvdoc.ExtractAllFromFS(elvFiles)
 	})
-	return docs
+	return docsMap
 }

pkg/mods/doc/doc_test.go

@@ -5,6 +5,7 @@ import (
 	"io/fs"
 	"testing"
 
+	"src.elv.sh/pkg/elvdoc"
 	"src.elv.sh/pkg/eval"
 	"src.elv.sh/pkg/eval/evaltest"
 	"src.elv.sh/pkg/md"
@@ -89,5 +90,5 @@ func setupDoc(ev *eval.Evaler) {
 }
 
 func render(s string, w int) string {
-	return md.RenderString(s, &md.TTYCodec{Width: w, HighlightCodeBlock: doc.HighlightCodeBlock})
+	return md.RenderString(s, &md.TTYCodec{Width: w, HighlightCodeBlock: elvdoc.HighlightCodeBlock})
 }

website/Makefile

@@ -40,6 +40,8 @@ endif
 # Don't remove intermediate targets
 .SECONDARY:
 
+# Rules below have dynamic prerequisite lists, which requires GNU Make's
+# .SECONDEXPANSION.
 .SECONDEXPANSION:
 
 tools/%.bin: cmd/% $$(wildcard cmd/%/*) go.mod ../go.mod $$(shell tools/cmd-deps ./cmd/%)

website/cmd/md2html/highlight.go

@@ -5,12 +5,12 @@ import (
 	"html"
 	"strings"
 
-	"src.elv.sh/pkg/mods/doc"
+	"src.elv.sh/pkg/elvdoc"
 	"src.elv.sh/pkg/ui"
 )
 
 func convertCodeBlock(info, code string) string {
-	return textToHTML(doc.HighlightCodeBlock(info, code))
+	return textToHTML(elvdoc.HighlightCodeBlock(info, code))
 }
 
 func textToHTML(t ui.Text) string {

website/cmd/md2html/macros.go

@@ -9,9 +9,14 @@ import (
 	"os"
 	"strings"
 
-	"src.elv.sh/pkg/mods/doc"
+	"src.elv.sh/pkg/elvdoc"
 )
 
+// Unlike Elvish's builtin documentation which embeds all the relevant .elv
+// files into the binary itself, we read the filesystem at runtime. This allows
+// us to read the new .elv file without rebuilding this program.
+var pkgFS = os.DirFS("../pkg")
+
 func filter(in io.Reader, out io.Writer) {
 	f := filterer{}
 	f.filter(in, out)
@@ -41,14 +46,16 @@ func (f *filterer) filter(in io.Reader, out io.Writer) {
 		fmt.Fprintln(out, line)
 	}
 	if f.module != "" {
-		ns := f.module + ":"
-		if f.module == "builtin" {
-			ns = ""
+		symbolPrefix := ""
+		if f.module != "builtin" {
+			symbolPrefix = f.module + ":"
+		}
+		docs, err := elvdoc.ExtractFromFS(pkgFS, symbolPrefix)
+		if err != nil {
+			log.Fatal(err)
 		}
-		docs := doc.Docs()[ns]
-
 		var buf bytes.Buffer
-		writeElvdocSections(&buf, ns, docs)
+		writeElvdocSections(&buf, symbolPrefix, docs)
 		filter(&buf, out)
 	}
 }

website/cmd/md2html/main.go

@@ -5,11 +5,16 @@
 //
 // It first applies a pre-processing step that expands the following macros:
 //
-//   - @module inserts elvdocs for a given module
+//   - @module declares the file to be the reference doc for a module.
 //
-//   - @ttyshot inserts a ttyshot
+//     This has two effects: it declares a docset anchor immediately, and causes
+//     the inserts elvdocs for the module at the end of the current to be
+//     inserted at the end of the file. It should appear at the beginning of the
+//     reference doc for a module.
 //
-//   - @dl expands to a binary download link
+//   - @ttyshot inserts a ttyshot.
+//
+//   - @dl expands to a binary download link.
 //
 // The processed Markdown source is then converted to HTML using a codec based
 // on [md.HTMLCodec], with the following additional features:
@@ -21,7 +26,7 @@
 //   - Table of content (optional, turn on with <!-- toc -->)
 //
 //   - Implicit links to elvdoc targets when link destination is empty and link
-//     text is code span - for example, [`put`]() has destination
+//     text is a single code span - for example, [`put`]() has destination
 //     builtin.html#put (or just #put within doc for the builtin module itself)
 //
 //   - Section numbers for headings (optional, turn on with <-- number-sections

website/tools/md-deps

@@ -6,3 +6,17 @@
 
 cat ${1%.html}.md |
   awk '$1 == "@ttyshot" { print $2 ".ttyshot.html" }'
+
+cat ${1%.html}.md |
+  awk '$1 == "@module" {
+         if ($2 == "builtin") {
+           print "eval"
+         } else if ($2 == "edit") {
+           print "eval"
+         } else {
+           print "mods/" $2
+         }
+       }' |
+  while read dir; do
+    echo ../pkg/$dir ../pkg/$dir/*.elv
+  done