12999379643

Committed 27 Jan 2025 10:16PM CUT coverage: 65.591%. Remained the same

Build # 12999379643

Build Type

push

github

Committed by

web-flow

Commit Message

Use Hugo fork with Mojo lexer for docs (#132)

Run Details

1098 of 1674 relevant lines covered (65.59%)

31.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.51

/document/doctest.go

package document

import (
        "bufio"
        "fmt"
        "os"
        "path"
        "path/filepath"
        "strings"
)

const docTestAttr = "doctest"
const hideAttr = "hide"
const globalAttr = "global"

func (proc *Processor) extractDocTests() error {
        proc.docTests = []*docTest{}
        return proc.walkDocs(proc.Docs, proc.extractTests, func(elem Named) string {
                return elem.GetFileName()
        })
}

func (proc *Processor) extractDocTestsMarkdown(baseDir string, build bool) error {
        proc.docTests = []*docTest{}
        outDir := filepath.Clean(proc.Config.OutputDir)
        baseDir = filepath.Clean(baseDir)
        err := filepath.WalkDir(baseDir,
                func(p string, info os.DirEntry, err error) error {
                        if err != nil {
                                return err
                        }
                        if info.IsDir() {
                                return nil
                        }
                        return proc.extractMarkdown(p, baseDir, outDir, build)
                })
        if err != nil {
                return err
        }
        if proc.Config.TestOutput != "" {
                err = proc.writeDocTests(proc.Config.TestOutput)
                if err != nil {
                        return err
                }
        }
        return nil
}

func (proc *Processor) extractMarkdown(file, baseDir, outDir string, build bool) error {
        if strings.HasSuffix(strings.ToLower(file), ".json") {
                return nil
        }

        cleanPath := path.Clean(file)
        relPath := filepath.Clean(strings.TrimPrefix(cleanPath, baseDir))
        targetPath := filepath.Join(outDir, relPath)
        targetDir, _ := filepath.Split(targetPath)

        content, err := os.ReadFile(cleanPath)
        if err != nil {
                return err
        }
        contentStr := string(content)
        if strings.HasSuffix(strings.ToLower(file), ".md") {
                var err error
                contentStr, err = proc.extractTests(contentStr, []string{strings.TrimSuffix(relPath, ".md")}, 1)
                if err != nil {
                        return err
                }
        }

        if build {
                err = proc.mkDirs(targetDir)
                if err != nil {
                        return err
                }
                return proc.WriteFile(targetPath, contentStr)
        }
        return nil
}

func (proc *Processor) writeDocTests(dir string) error {
        if dir == "" {
                return nil
        }
        for _, test := range proc.docTests {
                b := strings.Builder{}
                err := proc.Template.ExecuteTemplate(&b, "doctest.mojo", test)
                if err != nil {
                        return err
                }
                filePath := strings.Join(test.Path, "_")
                filePath += "_" + test.Name + "_test.mojo"
                fullPath := path.Join(dir, filePath)

                parentDir, _ := filepath.Split(filepath.Clean(fullPath))
                err = proc.mkDirs(parentDir)
                if err != nil {
                        return err
                }

                err = proc.WriteFile(fullPath, b.String())
                if err != nil {
                        return err
                }
        }
        fmt.Printf("Extracted %d tests.\n", len(proc.docTests))
        return nil
}

func (proc *Processor) extractTests(text string, elems []string, modElems int) (string, error) {
        t, tests, err := extractTestsText(text, elems, proc.Config.Strict)
        if err != nil {
                return "", err
        }
        proc.docTests = append(proc.docTests, tests...)
        return t, nil
}

func extractTestsText(text string, elems []string, strict bool) (string, []*docTest, error) {
        scanner := bufio.NewScanner(strings.NewReader(text))
        outText := strings.Builder{}

        fenced := false
        blocks := map[string]*docTest{}
        var blockLines []string
        var globalLines []string
        var blockName string
        var excluded bool
        var global bool
        var count int
        for scanner.Scan() {
                origLine := scanner.Text()

                isStart := false
                isFence := strings.HasPrefix(origLine, codeFence3)
                if isFence && !fenced {
                        var ok bool
                        var err error
                        blockName, excluded, global, ok, err = parseBlockAttr(origLine)
                        if err != nil {
                                if err := warnOrError(strict, "%s in %s", err.Error(), strings.Join(elems, ".")); err != nil {
                                        return "", nil, err
                                }
                        }
                        if !ok {
                                blockName = ""
                        }
                        fenced = true
                        isStart = true
                }

                if !excluded {
                        outText.WriteString(origLine)
                        outText.WriteRune('\n')
                }

                if fenced && !isFence && blockName != "" {
                        if global {
                                globalLines = append(globalLines, origLine)
                        } else {
                                blockLines = append(blockLines, origLine)
                        }
                }
                count++

                if isFence && fenced && !isStart {
                        if blockName == "" {
                                excluded = false
                                global = false
                                fenced = false
                                continue
                        }
                        if dt, ok := blocks[blockName]; ok {
                                dt.Code = append(dt.Code, blockLines...)
                                dt.Global = append(dt.Global, globalLines...)
                        } else {
                                blocks[blockName] = &docTest{
                                        Name:   blockName,
                                        Path:   elems,
                                        Code:   append([]string{}, blockLines...),
                                        Global: append([]string{}, globalLines...),
                                }
                        }
                        blockLines = blockLines[:0]
                        globalLines = globalLines[:0]
                        excluded = false
                        global = false
                        fenced = false
                }
        }
        if err := scanner.Err(); err != nil {
                panic(err)
        }
        if fenced {
                if err := warnOrError(strict, "unbalanced code block in %s", strings.Join(elems, ".")); err != nil {
                        return "", nil, err
                }
        }

        tests := make([]*docTest, 0, len(blocks))
        for _, block := range blocks {
                tests = append(tests, block)
        }

        return strings.TrimSuffix(outText.String(), "\n"), tests, nil
}

func parseBlockAttr(line string) (name string, hide bool, global bool, ok bool, err error) {
        parts := strings.SplitN(line, "{", 2)
        if len(parts) < 2 {
                return
        }
        attrString := strings.TrimSpace(parts[1])
        if !strings.HasSuffix(attrString, "}") {
                err = fmt.Errorf("missing closing parentheses in code block attributes")
                return
        }
        attrString = strings.TrimSuffix(attrString, "}")

        quoted := false
        attrPairs := strings.FieldsFunc(attrString, func(r rune) bool {
                if r == '"' {
                        quoted = !quoted
                }
                return !quoted && r == ' '
        })

        for _, pair := range attrPairs {
                elems := strings.Split(pair, "=")
                if len(elems) > 2 {
                        err = fmt.Errorf("malformed code block attributes '%s'", pair)
                        return
                }
                if len(elems) < 2 {
                        continue
                }

                key := strings.TrimSpace(elems[0])
                if key == docTestAttr {
                        name = strings.Trim(elems[1], "\"")
                        continue
                }
                if key == hideAttr {
                        h := strings.Trim(elems[1], "\" ")
                        if h == "true" {
                                hide = true
                        }
                        continue
                }
                if key == globalAttr {
                        g := strings.Trim(elems[1], "\"")
                        if g == "true" {
                                global = true
                        }
                        continue
                }
        }
        ok = true
        return
}

1	package document
2
3	import (
4	"bufio"
5	"fmt"
6	"os"
7	"path"
8	"path/filepath"
9	"strings"
10	)
11
12	const docTestAttr = "doctest"
13	const hideAttr = "hide"
14	const globalAttr = "global"
15
16	func (proc *Processor) extractDocTests() error {	6✔
17	proc.docTests = []*docTest{}	6✔
18	return proc.walkDocs(proc.Docs, proc.extractTests, func(elem Named) string {	53✔
19	return elem.GetFileName()	47✔
20	})	47✔
21	}
22
23	func (proc *Processor) extractDocTestsMarkdown(baseDir string, build bool) error {	2✔
24	proc.docTests = []*docTest{}	2✔
25	outDir := filepath.Clean(proc.Config.OutputDir)	2✔
26	baseDir = filepath.Clean(baseDir)	2✔
27	err := filepath.WalkDir(baseDir,	2✔
28	func(p string, info os.DirEntry, err error) error {	8✔
29	if err != nil {	6✔
30	return err	×
31	}	×
32	if info.IsDir() {	8✔
33	return nil	2✔
34	}	2✔
35	return proc.extractMarkdown(p, baseDir, outDir, build)	4✔
36	})
37	if err != nil {	2✔
38	return err	×
39	}	×
40	if proc.Config.TestOutput != "" {	3✔
41	err = proc.writeDocTests(proc.Config.TestOutput)	1✔
42	if err != nil {	1✔
43	return err	×
44	}	×
45	}
46	return nil	2✔
47	}
48
49	func (proc *Processor) extractMarkdown(file, baseDir, outDir string, build bool) error {	4✔
50	if strings.HasSuffix(strings.ToLower(file), ".json") {	5✔
51	return nil	1✔
52	}	1✔
53
54	cleanPath := path.Clean(file)	3✔
55	relPath := filepath.Clean(strings.TrimPrefix(cleanPath, baseDir))	3✔
56	targetPath := filepath.Join(outDir, relPath)	3✔
57	targetDir, _ := filepath.Split(targetPath)	3✔
58		3✔
59	content, err := os.ReadFile(cleanPath)	3✔
60	if err != nil {	3✔
61	return err	×
62	}	×
63	contentStr := string(content)	3✔
64	if strings.HasSuffix(strings.ToLower(file), ".md") {	5✔
65	var err error	2✔
66	contentStr, err = proc.extractTests(contentStr, []string{strings.TrimSuffix(relPath, ".md")}, 1)	2✔
67	if err != nil {	2✔
68	return err	×
69	}	×
70	}
71
72	if build {	6✔
73	err = proc.mkDirs(targetDir)	3✔
74	if err != nil {	3✔
75	return err	×
76	}	×
77	return proc.WriteFile(targetPath, contentStr)	3✔
78	}
79	return nil	×
80	}
81
82	func (proc *Processor) writeDocTests(dir string) error {	1✔
83	if dir == "" {	1✔
84	return nil	×
85	}	×
86	for _, test := range proc.docTests {	2✔
87	b := strings.Builder{}	1✔
88	err := proc.Template.ExecuteTemplate(&b, "doctest.mojo", test)	1✔
89	if err != nil {	1✔
90	return err	×
91	}	×
92	filePath := strings.Join(test.Path, "_")	1✔
93	filePath += "_" + test.Name + "_test.mojo"	1✔
94	fullPath := path.Join(dir, filePath)	1✔
95		1✔
96	parentDir, _ := filepath.Split(filepath.Clean(fullPath))	1✔
97	err = proc.mkDirs(parentDir)	1✔
98	if err != nil {	1✔
99	return err	×
100	}	×
101
102	err = proc.WriteFile(fullPath, b.String())	1✔
103	if err != nil {	1✔
104	return err	×
105	}	×
106	}
107	fmt.Printf("Extracted %d tests.\n", len(proc.docTests))	1✔
108	return nil	1✔
109	}
110
111	func (proc *Processor) extractTests(text string, elems []string, modElems int) (string, error) {	309✔
112	t, tests, err := extractTestsText(text, elems, proc.Config.Strict)	309✔
113	if err != nil {	309✔
114	return "", err	×
115	}	×
116	proc.docTests = append(proc.docTests, tests...)	309✔
117	return t, nil	309✔
118	}
119
120	func extractTestsText(text string, elems []string, strict bool) (string, []*docTest, error) {	309✔
121	scanner := bufio.NewScanner(strings.NewReader(text))	309✔
122	outText := strings.Builder{}	309✔
123		309✔
124	fenced := false	309✔
125	blocks := map[string]*docTest{}	309✔
126	var blockLines []string	309✔
127	var globalLines []string	309✔
128	var blockName string	309✔
129	var excluded bool	309✔
130	var global bool	309✔
131	var count int	309✔
132	for scanner.Scan() {	552✔
133	origLine := scanner.Text()	243✔
134		243✔
135	isStart := false	243✔
136	isFence := strings.HasPrefix(origLine, codeFence3)	243✔
137	if isFence && !fenced {	262✔
138	var ok bool	19✔
139	var err error	19✔
140	blockName, excluded, global, ok, err = parseBlockAttr(origLine)	19✔
141	if err != nil {	19✔
142	if err := warnOrError(strict, "%s in %s", err.Error(), strings.Join(elems, ".")); err != nil {	×
143	return "", nil, err	×
144	}	×
145	}
146	if !ok {	19✔
147	blockName = ""	×
148	}	×
149	fenced = true	19✔
150	isStart = true	19✔
151	}
152
153	if !excluded {	437✔
154	outText.WriteString(origLine)	194✔
155	outText.WriteRune('\n')	194✔
156	}	194✔
157
158	if fenced && !isFence && blockName != "" {	275✔
159	if global {	46✔
160	globalLines = append(globalLines, origLine)	14✔
161	} else {	32✔
162	blockLines = append(blockLines, origLine)	18✔
163	}	18✔
164	}
165	count++	243✔
166		243✔
167	if isFence && fenced && !isStart {	262✔
168	if blockName == "" {	19✔
169	excluded = false	×
170	global = false	×
171	fenced = false	×
172	continue	×
173	}
174	if dt, ok := blocks[blockName]; ok {	31✔
175	dt.Code = append(dt.Code, blockLines...)	12✔
176	dt.Global = append(dt.Global, globalLines...)	12✔
177	} else {	19✔
178	blocks[blockName] = &docTest{	7✔
179	Name: blockName,	7✔
180	Path: elems,	7✔
181	Code: append([]string{}, blockLines...),	7✔
182	Global: append([]string{}, globalLines...),	7✔
183	}	7✔
184	}	7✔
185	blockLines = blockLines[:0]	19✔
186	globalLines = globalLines[:0]	19✔
187	excluded = false	19✔
188	global = false	19✔
189	fenced = false	19✔
190	}
191	}
192	if err := scanner.Err(); err != nil {	309✔
193	panic(err)	×
194	}
195	if fenced {	309✔
196	if err := warnOrError(strict, "unbalanced code block in %s", strings.Join(elems, ".")); err != nil {	×
197	return "", nil, err	×
198	}	×
199	}
200
201	tests := make([]*docTest, 0, len(blocks))	309✔
202	for _, block := range blocks {	316✔
203	tests = append(tests, block)	7✔
204	}	7✔
205
206	return strings.TrimSuffix(outText.String(), "\n"), tests, nil	309✔
207	}
208
209	func parseBlockAttr(line string) (name string, hide bool, global bool, ok bool, err error) {	26✔
210	parts := strings.SplitN(line, "{", 2)	26✔
211	if len(parts) < 2 {	27✔
212	return	1✔
213	}	1✔
214	attrString := strings.TrimSpace(parts[1])	25✔
215	if !strings.HasSuffix(attrString, "}") {	25✔
216	err = fmt.Errorf("missing closing parentheses in code block attributes")	×
217	return	×
218	}	×
219	attrString = strings.TrimSuffix(attrString, "}")	25✔
220		25✔
221	quoted := false	25✔
222	attrPairs := strings.FieldsFunc(attrString, func(r rune) bool {	627✔
223	if r == '"' {	652✔
224	quoted = !quoted	50✔
225	}	50✔
226	return !quoted && r == ' '	602✔
227	})
228
229	for _, pair := range attrPairs {	73✔
230	elems := strings.Split(pair, "=")	48✔
231	if len(elems) > 2 {	48✔
232	err = fmt.Errorf("malformed code block attributes '%s'", pair)	×
233	return	×
234	}	×
235	if len(elems) < 2 {	50✔
236	continue	2✔
237	}
238
239	key := strings.TrimSpace(elems[0])	46✔
240	if key == docTestAttr {	70✔
241	name = strings.Trim(elems[1], "\"")	24✔
242	continue	24✔
243	}
244	if key == hideAttr {	37✔
245	h := strings.Trim(elems[1], "\" ")	15✔
246	if h == "true" {	30✔
247	hide = true	15✔
248	}	15✔
249	continue	15✔
250	}
251	if key == globalAttr {	13✔
252	g := strings.Trim(elems[1], "\"")	6✔
253	if g == "true" {	12✔
254	global = true	6✔
255	}	6✔
256	continue	6✔
257	}
258	}
259	ok = true	25✔
260	return	25✔
261	}

mlange-42 / modo / 12999379643

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous