12976434537

Committed 26 Jan 2025 04:31PM CUT coverage: 58.343% (-2.4%) from 60.763%

Build # 12976434537

Build Type

push

github

Committed by

web-flow

Commit Message

Doc-test markdown files (#119)

Run Details

8 of 90 new or added lines in 4 files covered. (8.89%)

2 existing lines in 1 file now uncovered.

993 of 1702 relevant lines covered (58.34%)

28.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.66

/document/doctest.go

package document

import (
        "bufio"
        "fmt"
        "os"
        "path"
        "path/filepath"
        "strings"
)

const docTestAttr = "doctest"
const hideAttr = "hide"
const globalAttr = "global"

func (proc *Processor) extractDocTests() error {
        proc.docTests = []*docTest{}
        return proc.walkDocs(proc.Docs, proc.extractTests, func(elem Named) string {
                return elem.GetFileName()
        })
}

func (proc *Processor) extractDocTestsMarkdown(baseDir string, build bool) error {
        proc.docTests = []*docTest{}
        outDir := filepath.Clean(proc.Config.OutputDir)
        baseDir = filepath.Clean(baseDir)
        err := filepath.WalkDir(baseDir,
                func(p string, info os.DirEntry, err error) error {
                        if err != nil {
                                return err
                        }
                        if info.IsDir() {
                                return nil
                        }
                        return proc.extractMarkdown(p, baseDir, outDir, build)
                })
        if err != nil {
                return err
        }
        if proc.Config.TestOutput != "" {
                err = proc.writeDocTests(proc.Config.TestOutput)
                if err != nil {
                        return err
                }
        }
        return nil
}

func (proc *Processor) extractMarkdown(file, baseDir, outDir string, build bool) error {
        if strings.HasSuffix(strings.ToLower(file), ".json") {
                return nil
        }

        cleanPath := path.Clean(file)
        relPath := filepath.Clean(strings.TrimPrefix(cleanPath, baseDir))
        targetPath := filepath.Join(outDir, relPath)
        targetDir, _ := filepath.Split(targetPath)

        content, err := os.ReadFile(cleanPath)
        if err != nil {
                return err
        }
        contentStr := string(content)
        if strings.HasSuffix(strings.ToLower(file), ".md") {
                var err error
                contentStr, err = proc.extractTests(contentStr, []string{strings.TrimSuffix(relPath, ".md")}, 1)
                if err != nil {
                        return err
                }
        }

        if build {
                err = proc.mkDirs(targetDir)
                if err != nil {
                        return err
                }
                return proc.WriteFile(targetPath, contentStr)
        }
        return nil
}

func (proc *Processor) writeDocTests(dir string) error {
        if dir == "" {
                return nil
        }
        for _, test := range proc.docTests {
                b := strings.Builder{}
                err := proc.Template.ExecuteTemplate(&b, "doctest.mojo", test)
                if err != nil {
                        return err
                }
                filePath := strings.Join(test.Path, "_")
                filePath += "_" + test.Name + "_test.mojo"
                fullPath := path.Join(dir, filePath)

                parentDir, _ := filepath.Split(filepath.Clean(fullPath))
                err = proc.mkDirs(parentDir)
                if err != nil {
                        return err
                }

                err = proc.WriteFile(fullPath, b.String())
                if err != nil {
                        return err
                }
        }
        fmt.Printf("Extracted %d tests.\n", len(proc.docTests))
        return nil
}

func (proc *Processor) extractTests(text string, elems []string, modElems int) (string, error) {
        t, tests, err := extractTestsText(text, elems, proc.Config.Strict)
        if err != nil {
                return "", err
        }
        proc.docTests = append(proc.docTests, tests...)
        return t, nil
}

func extractTestsText(text string, elems []string, strict bool) (string, []*docTest, error) {
        scanner := bufio.NewScanner(strings.NewReader(text))
        outText := strings.Builder{}

        fenced := false
        blocks := map[string]*docTest{}
        var blockLines []string
        var globalLines []string
        var blockName string
        var excluded bool
        var global bool
        var count int
        for scanner.Scan() {
                origLine := scanner.Text()

                isStart := false
                isFence := strings.HasPrefix(origLine, codeFence3)
                if isFence && !fenced {
                        var ok bool
                        var err error
                        blockName, excluded, global, ok, err = parseBlockAttr(origLine)
                        if err != nil {
                                if err := warnOrError(strict, "%s in %s", err.Error(), strings.Join(elems, ".")); err != nil {
                                        return "", nil, err
                                }
                        }
                        if !ok {
                                blockName = ""
                        }
                        fenced = true
                        isStart = true
                }

                if !excluded {
                        outText.WriteString(origLine)
                        outText.WriteRune('\n')
                }

                if fenced && !isFence && blockName != "" {
                        if global {
                                globalLines = append(globalLines, origLine)
                        } else {
                                blockLines = append(blockLines, origLine)
                        }
                }
                count++

                if isFence && fenced && !isStart {
                        if blockName == "" {
                                excluded = false
                                global = false
                                fenced = false
                                continue
                        }
                        if dt, ok := blocks[blockName]; ok {
                                dt.Code = append(dt.Code, blockLines...)
                                dt.Global = append(dt.Global, globalLines...)
                        } else {
                                blocks[blockName] = &docTest{
                                        Name:   blockName,
                                        Path:   elems,
                                        Code:   append([]string{}, blockLines...),
                                        Global: append([]string{}, globalLines...),
                                }
                        }
                        blockLines = blockLines[:0]
                        globalLines = globalLines[:0]
                        excluded = false
                        global = false
                        fenced = false
                }
        }
        if err := scanner.Err(); err != nil {
                panic(err)
        }
        if fenced {
                if err := warnOrError(strict, "unbalanced code block in %s", strings.Join(elems, ".")); err != nil {
                        return "", nil, err
                }
        }

        tests := make([]*docTest, 0, len(blocks))
        for _, block := range blocks {
                tests = append(tests, block)
        }

        return strings.TrimSuffix(outText.String(), "\n"), tests, nil
}

func parseBlockAttr(line string) (name string, hide bool, global bool, ok bool, err error) {
        parts := strings.SplitN(line, "{", 2)
        if len(parts) < 2 {
                return
        }
        attrString := strings.TrimSpace(parts[1])
        if !strings.HasSuffix(attrString, "}") {
                err = fmt.Errorf("missing closing parentheses in code block attributes")
                return
        }
        attrString = strings.TrimSuffix(attrString, "}")

        quoted := false
        attrPairs := strings.FieldsFunc(attrString, func(r rune) bool {
                if r == '"' {
                        quoted = !quoted
                }
                return !quoted && r == ' '
        })

        for _, pair := range attrPairs {
                elems := strings.Split(pair, "=")
                if len(elems) > 2 {
                        err = fmt.Errorf("malformed code block attributes '%s'", pair)
                        return
                }
                if len(elems) < 2 {
                        continue
                }

                key := strings.TrimSpace(elems[0])
                if key == docTestAttr {
                        name = strings.Trim(elems[1], "\"")
                        continue
                }
                if key == hideAttr {
                        h := strings.Trim(elems[1], "\" ")
                        if h == "true" {
                                hide = true
                        }
                        continue
                }
                if key == globalAttr {
                        g := strings.Trim(elems[1], "\"")
                        if g == "true" {
                                global = true
                        }
                        continue
                }
        }
        ok = true
        return
}

1	package document
2
3	import (
4	"bufio"
5	"fmt"
6	"os"
7	"path"
8	"path/filepath"
9	"strings"
10	)
11
12	const docTestAttr = "doctest"
13	const hideAttr = "hide"
14	const globalAttr = "global"
15
16	func (proc *Processor) extractDocTests() error {	6✔
17	proc.docTests = []*docTest{}	6✔
18	return proc.walkDocs(proc.Docs, proc.extractTests, func(elem Named) string {	51✔
19	return elem.GetFileName()	45✔
20	})	45✔
21	}
22
NEW 23	func (proc *Processor) extractDocTestsMarkdown(baseDir string, build bool) error {	×
NEW 24	proc.docTests = []*docTest{}	×
NEW 25	outDir := filepath.Clean(proc.Config.OutputDir)	×
NEW 26	baseDir = filepath.Clean(baseDir)	×
NEW 27	err := filepath.WalkDir(baseDir,	×
NEW 28	func(p string, info os.DirEntry, err error) error {	×
NEW 29	if err != nil {	×
NEW 30	return err	×
NEW 31	}	×
NEW 32	if info.IsDir() {	×
NEW 33	return nil	×
NEW 34	}	×
NEW 35	return proc.extractMarkdown(p, baseDir, outDir, build)	×
36	})
NEW 37	if err != nil {	×
NEW 38	return err	×
NEW 39	}	×
NEW 40	if proc.Config.TestOutput != "" {	×
NEW 41	err = proc.writeDocTests(proc.Config.TestOutput)	×
NEW 42	if err != nil {	×
NEW 43	return err	×
NEW 44	}	×
45	}
NEW 46	return nil	×
47	}
48
NEW 49	func (proc *Processor) extractMarkdown(file, baseDir, outDir string, build bool) error {	×
NEW 50	if strings.HasSuffix(strings.ToLower(file), ".json") {	×
51	return nil	×
52	}	×
53
NEW 54	cleanPath := path.Clean(file)	×
NEW 55	relPath := filepath.Clean(strings.TrimPrefix(cleanPath, baseDir))	×
NEW 56	targetPath := filepath.Join(outDir, relPath)	×
NEW 57	targetDir, _ := filepath.Split(targetPath)	×
NEW 58		×
NEW 59	content, err := os.ReadFile(cleanPath)	×
60	if err != nil {	×
61	return err	×
62	}	×
NEW 63	contentStr := string(content)	×
NEW 64	if strings.HasSuffix(strings.ToLower(file), ".md") {	×
NEW 65	var err error	×
NEW 66	contentStr, err = proc.extractTests(contentStr, []string{strings.TrimSuffix(relPath, ".md")}, 1)	×
NEW 67	if err != nil {	×
NEW 68	return err	×
NEW 69	}	×
70	}
71
NEW 72	if build {	×
NEW 73	err = proc.mkDirs(targetDir)	×
NEW 74	if err != nil {	×
NEW 75	return err	×
NEW 76	}	×
NEW 77	return proc.WriteFile(targetPath, contentStr)	×
78	}
NEW 79	return nil	×
80	}
81
NEW 82	func (proc *Processor) writeDocTests(dir string) error {	×
NEW 83	if dir == "" {	×
NEW 84	return nil	×
NEW 85	}	×
86	for _, test := range proc.docTests {	×
87	b := strings.Builder{}	×
88	err := proc.Template.ExecuteTemplate(&b, "doctest.mojo", test)	×
89	if err != nil {	×
90	return err	×
91	}	×
92	filePath := strings.Join(test.Path, "_")	×
93	filePath += "_" + test.Name + "_test.mojo"	×
94	fullPath := path.Join(dir, filePath)	×
95		×
NEW 96	parentDir, _ := filepath.Split(filepath.Clean(fullPath))	×
NEW 97	err = proc.mkDirs(parentDir)	×
NEW 98	if err != nil {	×
NEW 99	return err	×
NEW 100	}	×
101
102	err = proc.WriteFile(fullPath, b.String())	×
103	if err != nil {	×
104	return err	×
105	}	×
106	}
107	fmt.Printf("Extracted %d tests.\n", len(proc.docTests))	×
108	return nil	×
109	}
110
111	func (proc *Processor) extractTests(text string, elems []string, modElems int) (string, error) {	295✔
112	t, tests, err := extractTestsText(text, elems, proc.Config.Strict)	295✔
113	if err != nil {	295✔
NEW 114	return "", err	×
NEW 115	}	×
116	proc.docTests = append(proc.docTests, tests...)	295✔
117	return t, nil	295✔
118	}
119
120	func extractTestsText(text string, elems []string, strict bool) (string, []*docTest, error) {	295✔
121	scanner := bufio.NewScanner(strings.NewReader(text))	295✔
122	outText := strings.Builder{}	295✔
123		295✔
124	fenced := false	295✔
125	blocks := map[string]*docTest{}	295✔
126	var blockLines []string	295✔
127	var globalLines []string	295✔
128	var blockName string	295✔
129	var excluded bool	295✔
130	var global bool	295✔
131	var count int	295✔
132	for scanner.Scan() {	506✔
133	origLine := scanner.Text()	211✔
134		211✔
135	isStart := false	211✔
136	isFence := strings.HasPrefix(origLine, codeFence3)	211✔
137	if isFence && !fenced {	225✔
138	var ok bool	14✔
139	var err error	14✔
140	blockName, excluded, global, ok, err = parseBlockAttr(origLine)	14✔
141	if err != nil {	14✔
NEW 142	if err := warnOrError(strict, "%s in %s", err.Error(), strings.Join(elems, ".")); err != nil {	×
NEW 143	return "", nil, err	×
UNCOV 144	}	×
145	}
146	if !ok {	14✔
147	blockName = ""	×
148	}	×
149	fenced = true	14✔
150	isStart = true	14✔
151	}
152
153	if !excluded {	386✔
154	outText.WriteString(origLine)	175✔
155	outText.WriteRune('\n')	175✔
156	}	175✔
157
158	if fenced && !isFence && blockName != "" {	234✔
159	if global {	33✔
160	globalLines = append(globalLines, origLine)	10✔
161	} else {	23✔
162	blockLines = append(blockLines, origLine)	13✔
163	}	13✔
164	}
165	count++	211✔
166		211✔
167	if isFence && fenced && !isStart {	225✔
168	if blockName == "" {	14✔
169	excluded = false	×
170	global = false	×
171	fenced = false	×
172	continue	×
173	}
174	if dt, ok := blocks[blockName]; ok {	23✔
175	dt.Code = append(dt.Code, blockLines...)	9✔
176	dt.Global = append(dt.Global, globalLines...)	9✔
177	} else {	14✔
178	blocks[blockName] = &docTest{	5✔
179	Name: blockName,	5✔
180	Path: elems,	5✔
181	Code: append([]string{}, blockLines...),	5✔
182	Global: append([]string{}, globalLines...),	5✔
183	}	5✔
184	}	5✔
185	blockLines = blockLines[:0]	14✔
186	globalLines = globalLines[:0]	14✔
187	excluded = false	14✔
188	global = false	14✔
189	fenced = false	14✔
190	}
191	}
192	if err := scanner.Err(); err != nil {	295✔
193	panic(err)	×
194	}
195	if fenced {	295✔
NEW 196	if err := warnOrError(strict, "unbalanced code block in %s", strings.Join(elems, ".")); err != nil {	×
NEW 197	return "", nil, err	×
UNCOV 198	}	×
199	}
200
201	tests := make([]*docTest, 0, len(blocks))	295✔
202	for _, block := range blocks {	300✔
203	tests = append(tests, block)	5✔
204	}	5✔
205
206	return strings.TrimSuffix(outText.String(), "\n"), tests, nil	295✔
207	}
208
209	func parseBlockAttr(line string) (name string, hide bool, global bool, ok bool, err error) {	21✔
210	parts := strings.SplitN(line, "{", 2)	21✔
211	if len(parts) < 2 {	22✔
212	return	1✔
213	}	1✔
214	attrString := strings.TrimSpace(parts[1])	20✔
215	if !strings.HasSuffix(attrString, "}") {	20✔
216	err = fmt.Errorf("missing closing parentheses in code block attributes")	×
217	return	×
218	}	×
219	attrString = strings.TrimSuffix(attrString, "}")	20✔
220		20✔
221	quoted := false	20✔
222	attrPairs := strings.FieldsFunc(attrString, func(r rune) bool {	493✔
223	if r == '"' {	513✔
224	quoted = !quoted	40✔
225	}	40✔
226	return !quoted && r == ' '	473✔
227	})
228
229	for _, pair := range attrPairs {	59✔
230	elems := strings.Split(pair, "=")	39✔
231	if len(elems) > 2 {	39✔
232	err = fmt.Errorf("malformed code block attributes '%s'", pair)	×
233	return	×
234	}	×
235	if len(elems) < 2 {	41✔
236	continue	2✔
237	}
238
239	key := strings.TrimSpace(elems[0])	37✔
240	if key == docTestAttr {	56✔
241	name = strings.Trim(elems[1], "\"")	19✔
242	continue	19✔
243	}
244	if key == hideAttr {	30✔
245	h := strings.Trim(elems[1], "\" ")	12✔
246	if h == "true" {	24✔
247	hide = true	12✔
248	}	12✔
249	continue	12✔
250	}
251	if key == globalAttr {	11✔
252	g := strings.Trim(elems[1], "\"")	5✔
253	if g == "true" {	10✔
254	global = true	5✔
255	}	5✔
256	continue	5✔
257	}
258	}
259	ok = true	20✔
260	return	20✔
261	}

mlange-42 / modo / 12976434537

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous