From 93df41f506fb0c3ca897def5d62abaedc93323d8 Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 5 Oct 2022 19:55:36 +0100 Subject: [PATCH] Fix slight bug in katex (#21171) There is a small bug in #20571 whereby `$a a$b b$` will not be correctly detected as a math inline block of `a a$b b`. This PR fixes this. Also reenable test cases as per #21340 Signed-off-by: Andrew Thornton Co-authored-by: wxiaoguang --- modules/markup/html_test.go | 4 +- modules/markup/markdown/markdown_test.go | 50 ++++++++++ modules/markup/markdown/math/inline_parser.go | 47 ++++++--- modules/markup/markdown/meta.go | 4 +- modules/markup/markdown/meta_test.go | 4 +- modules/markup/markdown/renderconfig.go | 96 ++++++++++--------- modules/markup/markdown/renderconfig_test.go | 17 ++-- 7 files changed, 153 insertions(+), 69 deletions(-) diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 370bc21822..e57187a677 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -7,6 +7,7 @@ package markup_test import ( "context" "io" + "os" "strings" "testing" @@ -32,6 +33,7 @@ func TestMain(m *testing.M) { if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } + os.Exit(m.Run()) } func TestRender_Commits(t *testing.T) { @@ -336,7 +338,7 @@ func TestRender_emoji(t *testing.T) { `

Some text with 😄😄 2 emoji next to each other

`) test( "😎ðŸĪŠðŸ”ðŸĪ‘❓", - `

😎ðŸĪŠðŸ”ðŸĪ‘❓

`) + `

😎ðŸĪŠðŸ”ðŸĪ‘❓

`) // should match nothing test( diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index fdbc291c94..49ed3d75d6 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -6,6 +6,7 @@ package markdown_test import ( "context" + "os" "strings" "testing" @@ -37,6 +38,7 @@ func TestMain(m *testing.M) { if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } + os.Exit(m.Run()) } func TestRender_StandardLinks(t *testing.T) { @@ -426,3 +428,51 @@ func TestRenderEmojiInLinks_Issue12331(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, res) } + +func TestMathBlock(t *testing.T) { + const nl = "\n" + testcases := []struct { + testcase string + expected string + }{ + { + "$a$", + `

a

` + nl, + }, + { + "$ a $", + `

a

` + nl, + }, + { + "$a$ $b$", + `

a b

` + nl, + }, + { + `\(a\) \(b\)`, + `

a b

` + nl, + }, + { + `$a a$b b$`, + `

a a$b b

` + nl, + }, + { + `a a$b b`, + `

a a$b b

` + nl, + }, + { + `a$b $a a$b b$`, + `

a$b a a$b b

` + nl, + }, + { + "$$a$$", + `
a
` + nl, + }, + } + + for _, test := range testcases { + res, err := RenderString(&markup.RenderContext{}, test.testcase) + assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) + assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase) + + } +} diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index 0339674b6c..8dc88eb858 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -37,7 +37,7 @@ func NewInlineBracketParser() parser.InlineParser { return defaultInlineBracketParser } -// Trigger triggers this parser on $ +// Trigger triggers this parser on $ or \ func (parser *inlineParser) Trigger() []byte { return parser.start[0:1] } @@ -50,29 +50,50 @@ func isAlphanumeric(b byte) bool { // Parse parses the current line and returns a result of parsing. func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { line, _ := block.PeekLine() - opener := bytes.Index(line, parser.start) - if opener < 0 { - return nil - } - if opener != 0 && isAlphanumeric(line[opener-1]) { + + if !bytes.HasPrefix(line, parser.start) { + // We'll catch this one on the next time round return nil } - opener += len(parser.start) - ender := bytes.Index(line[opener:], parser.end) - if ender < 0 { + precedingCharacter := block.PrecendingCharacter() + if precedingCharacter < 256 && isAlphanumeric(byte(precedingCharacter)) { + // need to exclude things like `a$` from being considered a start return nil } - if len(line) > opener+ender+len(parser.end) && isAlphanumeric(line[opener+ender+len(parser.end)]) { - return nil + + // move the opener marker point at the start of the text + opener := len(parser.start) + + // Now look for an ending line + ender := opener + for { + pos := bytes.Index(line[ender:], parser.end) + if pos < 0 { + return nil + } + + ender += pos + + // Now we want to check the character at the end of our parser section + // that is ender + len(parser.end) + pos = ender + len(parser.end) + if len(line) <= pos { + break + } + if !isAlphanumeric(line[pos]) { + break + } + // move the pointer onwards + ender += len(parser.end) } block.Advance(opener) _, pos := block.Position() node := NewInline() - segment := pos.WithStop(pos.Start + ender) + segment := pos.WithStop(pos.Start + ender - opener) node.AppendChild(node, ast.NewRawTextSegment(segment)) - block.Advance(ender + len(parser.end)) + block.Advance(ender - opener + len(parser.end)) trimBlock(node, block) return node diff --git a/modules/markup/markdown/meta.go b/modules/markup/markdown/meta.go index b08121e868..45d79d537a 100644 --- a/modules/markup/markdown/meta.go +++ b/modules/markup/markdown/meta.go @@ -88,7 +88,9 @@ func ExtractMetadataBytes(contents []byte, out interface{}) ([]byte, error) { line := contents[start:end] if isYAMLSeparator(line) { front = contents[frontMatterStart:start] - body = contents[end+1:] + if end+1 < len(contents) { + body = contents[end+1:] + } break } } diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 9332b35b42..720d0066f4 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -61,7 +61,7 @@ func TestExtractMetadataBytes(t *testing.T) { var meta structs.IssueTemplate body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) assert.NoError(t, err) - assert.Equal(t, bodyTest, body) + assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) assert.True(t, validateMetadata(meta)) }) @@ -82,7 +82,7 @@ func TestExtractMetadataBytes(t *testing.T) { var meta structs.IssueTemplate body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) assert.NoError(t, err) - assert.Equal(t, "", body) + assert.Equal(t, "", string(body)) assert.Equal(t, metaTest, meta) assert.True(t, validateMetadata(meta)) }) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index 003579115f..1ba75dbb68 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -5,10 +5,9 @@ package markdown import ( + "fmt" "strings" - "code.gitea.io/gitea/modules/log" - "github.com/yuin/goldmark/ast" "gopkg.in/yaml.v3" ) @@ -33,17 +32,13 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { } rc.yamlNode = value - type basicRenderConfig struct { - Gitea *yaml.Node `yaml:"gitea"` - TOC bool `yaml:"include_toc"` - Lang string `yaml:"lang"` + type commonRenderConfig struct { + TOC bool `yaml:"include_toc"` + Lang string `yaml:"lang"` } - - var basic basicRenderConfig - - err := value.Decode(&basic) - if err != nil { - return err + var basic commonRenderConfig + if err := value.Decode(&basic); err != nil { + return fmt.Errorf("unable to decode into commonRenderConfig %w", err) } if basic.Lang != "" { @@ -51,14 +46,48 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { } rc.TOC = basic.TOC - if basic.Gitea == nil { + + type controlStringRenderConfig struct { + Gitea string `yaml:"gitea"` + } + + var stringBasic controlStringRenderConfig + + if err := value.Decode(&stringBasic); err == nil { + if stringBasic.Gitea != "" { + switch strings.TrimSpace(strings.ToLower(stringBasic.Gitea)) { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + default: // "details" + rc.Meta = "details" + } + } return nil } - var control *string - if err := basic.Gitea.Decode(&control); err == nil && control != nil { - log.Info("control %v", control) - switch strings.TrimSpace(strings.ToLower(*control)) { + type giteaControl struct { + Meta *string `yaml:"meta"` + Icon *string `yaml:"details_icon"` + TOC *bool `yaml:"include_toc"` + Lang *string `yaml:"lang"` + } + + type complexGiteaConfig struct { + Gitea *giteaControl `yaml:"gitea"` + } + var complex complexGiteaConfig + if err := value.Decode(&complex); err != nil { + return fmt.Errorf("unable to decode into complexRenderConfig %w", err) + } + + if complex.Gitea == nil { + return nil + } + + if complex.Gitea.Meta != nil { + switch strings.TrimSpace(strings.ToLower(*complex.Gitea.Meta)) { case "none": rc.Meta = "none" case "table": @@ -66,39 +95,18 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { default: // "details" rc.Meta = "details" } - return nil } - type giteaControl struct { - Meta string `yaml:"meta"` - Icon string `yaml:"details_icon"` - TOC *yaml.Node `yaml:"include_toc"` - Lang string `yaml:"lang"` + if complex.Gitea.Icon != nil { + rc.Icon = strings.TrimSpace(strings.ToLower(*complex.Gitea.Icon)) } - var controlStruct *giteaControl - if err := basic.Gitea.Decode(controlStruct); err != nil || controlStruct == nil { - return err + if complex.Gitea.Lang != nil && *complex.Gitea.Lang != "" { + rc.Lang = *complex.Gitea.Lang } - switch strings.TrimSpace(strings.ToLower(controlStruct.Meta)) { - case "none": - rc.Meta = "none" - case "table": - rc.Meta = "table" - default: // "details" - rc.Meta = "details" - } - - rc.Icon = strings.TrimSpace(strings.ToLower(controlStruct.Icon)) - - if controlStruct.Lang != "" { - rc.Lang = controlStruct.Lang - } - - var toc bool - if err := controlStruct.TOC.Decode(&toc); err == nil { - rc.TOC = toc + if complex.Gitea.TOC != nil { + rc.TOC = *complex.Gitea.TOC } return nil diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index 1027035cda..672edbf46d 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -5,6 +5,7 @@ package markdown import ( + "strings" "testing" "gopkg.in/yaml.v3" @@ -81,9 +82,9 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { TOC: true, Lang: "testlang", }, ` - include_toc: true - lang: testlang -`, + include_toc: true + lang: testlang + `, }, { "complexlang", &RenderConfig{ @@ -91,9 +92,9 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Icon: "table", Lang: "testlang", }, ` - gitea: - lang: testlang -`, + gitea: + lang: testlang + `, }, { "complexlang2", &RenderConfig{ @@ -140,8 +141,8 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Icon: "table", Lang: "", } - if err := yaml.Unmarshal([]byte(tt.args), got); err != nil { - t.Errorf("RenderConfig.UnmarshalYAML() error = %v", err) + if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got); err != nil { + t.Errorf("RenderConfig.UnmarshalYAML() error = %v\n%q", err, tt.args) return }