PLT-3383: image proxy support (#7991)

* image proxy support

* go vet fix, remove mistakenly added coverage file

* fix test compile error

* add validation to config settings and documentation to model functions

* add message_source field to post

api/post.go

@@ -135,7 +135,7 @@ func saveIsPinnedPost(c *Context, w http.ResponseWriter, r *http.Request, isPinn
 			rpost := result.Data.(*model.Post)
 
 			message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", rpost.ChannelId, "", nil)
-			message.Add("post", rpost.ToJson())
+			message.Add("post", c.App.PostWithProxyAddedToImageURLs(rpost).ToJson())
 
 			c.App.Go(func() {
 				c.App.Publish(message)

api4/channel.go

@@ -386,7 +386,7 @@ func getPinnedPosts(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	} else {
 		w.Header().Set(model.HEADER_ETAG_SERVER, posts.Etag())
-		w.Write([]byte(posts.ToJson()))
+		w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson()))
 	}
 }
 

api4/post.go

@@ -56,7 +56,7 @@ func createPost(c *Context, w http.ResponseWriter, r *http.Request) {
 		post.CreateAt = 0
 	}
 
-	rp, err := c.App.CreatePostAsUser(post)
+	rp, err := c.App.CreatePostAsUser(c.App.PostWithProxyRemovedFromImageURLs(post))
 	if err != nil {
 		c.Err = err
 		return
@@ -66,7 +66,7 @@ func createPost(c *Context, w http.ResponseWriter, r *http.Request) {
 	c.App.UpdateLastActivityAtIfNeeded(c.Session)
 
 	w.WriteHeader(http.StatusCreated)
-	w.Write([]byte(rp.ToJson()))
+	w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rp).ToJson()))
 }
 
 func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -135,7 +135,7 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) {
 	if len(etag) > 0 {
 		w.Header().Set(model.HEADER_ETAG_SERVER, etag)
 	}
-	w.Write([]byte(list.ToJson()))
+	w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(list).ToJson()))
 }
 
 func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -168,7 +168,7 @@ func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	w.Write([]byte(posts.ToJson()))
+	w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson()))
 }
 
 func getPost(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -206,7 +206,7 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	} else {
 		w.Header().Set(model.HEADER_ETAG_SERVER, post.Etag())
-		w.Write([]byte(post.ToJson()))
+		w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(post).ToJson()))
 	}
 }
 
@@ -272,7 +272,7 @@ func getPostThread(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	} else {
 		w.Header().Set(model.HEADER_ETAG_SERVER, list.Etag())
-		w.Write([]byte(list.ToJson()))
+		w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(list).ToJson()))
 	}
 }
 
@@ -313,7 +313,7 @@ func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 
 	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
-	w.Write([]byte(posts.ToJson()))
+	w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson()))
 }
 
 func updatePost(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -341,13 +341,13 @@ func updatePost(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	post.Id = c.Params.PostId
 
-	rpost, err := c.App.UpdatePost(post, false)
+	rpost, err := c.App.UpdatePost(c.App.PostWithProxyRemovedFromImageURLs(post), false)
 	if err != nil {
 		c.Err = err
 		return
 	}
 
-	w.Write([]byte(rpost.ToJson()))
+	w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rpost).ToJson()))
 }
 
 func patchPost(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -373,13 +373,13 @@ func patchPost(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	patchedPost, err := c.App.PatchPost(c.Params.PostId, post)
+	patchedPost, err := c.App.PatchPost(c.Params.PostId, c.App.PostPatchWithProxyRemovedFromImageURLs(post))
 	if err != nil {
 		c.Err = err
 		return
 	}
 
-	w.Write([]byte(patchedPost.ToJson()))
+	w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(patchedPost).ToJson()))
 }
 
 func saveIsPinnedPost(c *Context, w http.ResponseWriter, r *http.Request, isPinned bool) {

app/notification.go

@@ -270,7 +270,7 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod
 	}
 
 	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POSTED, "", post.ChannelId, "", nil)
-	message.Add("post", post.ToJson())
+	message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
 	message.Add("channel_type", channel.Type)
 	message.Add("channel_display_name", channelName)
 	message.Add("channel_name", channel.Name)

app/post.go

@@ -4,6 +4,11 @@
 package app
 
 import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -309,7 +314,7 @@ func (a *App) SendEphemeralPost(userId string, post *model.Post) *model.Post {
 	}
 
 	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil)
-	message.Add("post", post.ToJson())
+	message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
 
 	a.Go(func() {
 		a.Publish(message)
@@ -419,7 +424,7 @@ func (a *App) PatchPost(postId string, patch *model.PostPatch) (*model.Post, *mo
 
 func (a *App) sendUpdatedPostEvent(post *model.Post) {
 	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
-	message.Add("post", post.ToJson())
+	message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
 
 	a.Go(func() {
 		a.Publish(message)
@@ -562,7 +567,7 @@ func (a *App) DeletePost(postId string) (*model.Post, *model.AppError) {
 		}
 
 		message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_DELETED, "", post.ChannelId, "", nil)
-		message.Add("post", post.ToJson())
+		message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
 
 		a.Go(func() {
 			a.Publish(message)
@@ -823,3 +828,124 @@ func (a *App) DoPostAction(postId string, actionId string, userId string) *model
 
 	return nil
 }
+
+func (a *App) PostListWithProxyAddedToImageURLs(list *model.PostList) *model.PostList {
+	if f := a.ImageProxyAdder(); f != nil {
+		return list.WithRewrittenImageURLs(f)
+	}
+	return list
+}
+
+func (a *App) PostWithProxyAddedToImageURLs(post *model.Post) *model.Post {
+	if f := a.ImageProxyAdder(); f != nil {
+		return post.WithRewrittenImageURLs(f)
+	}
+	return post
+}
+
+func (a *App) PostWithProxyRemovedFromImageURLs(post *model.Post) *model.Post {
+	if f := a.ImageProxyRemover(); f != nil {
+		return post.WithRewrittenImageURLs(f)
+	}
+	return post
+}
+
+func (a *App) PostPatchWithProxyRemovedFromImageURLs(patch *model.PostPatch) *model.PostPatch {
+	if f := a.ImageProxyRemover(); f != nil {
+		return patch.WithRewrittenImageURLs(f)
+	}
+	return patch
+}
+
+func (a *App) imageProxyConfig() (proxyType, proxyURL, options, siteURL string) {
+	cfg := a.Config()
+
+	if cfg.ServiceSettings.ImageProxyURL == nil || cfg.ServiceSettings.ImageProxyType == nil || cfg.ServiceSettings.SiteURL == nil {
+		return
+	}
+
+	proxyURL = *cfg.ServiceSettings.ImageProxyURL
+	proxyType = *cfg.ServiceSettings.ImageProxyType
+	siteURL = *cfg.ServiceSettings.SiteURL
+
+	if proxyURL == "" || proxyType == "" {
+		return "", "", "", ""
+	}
+
+	if proxyURL[len(proxyURL)-1] != '/' {
+		proxyURL += "/"
+	}
+
+	if cfg.ServiceSettings.ImageProxyOptions != nil {
+		options = *cfg.ServiceSettings.ImageProxyOptions
+	}
+
+	return
+}
+
+func (a *App) ImageProxyAdder() func(string) string {
+	proxyType, proxyURL, options, siteURL := a.imageProxyConfig()
+	if proxyType == "" {
+		return nil
+	}
+
+	return func(url string) string {
+		if strings.HasPrefix(url, proxyURL) {
+			return url
+		}
+
+		if url[0] == '/' {
+			url = siteURL + url
+		}
+
+		switch proxyType {
+		case "atmos/camo":
+			mac := hmac.New(sha1.New, []byte(options))
+			mac.Write([]byte(url))
+			digest := hex.EncodeToString(mac.Sum(nil))
+			return proxyURL + digest + "/" + hex.EncodeToString([]byte(url))
+		case "willnorris/imageproxy":
+			options := strings.Split(options, "|")
+			if len(options) > 1 {
+				mac := hmac.New(sha256.New, []byte(options[1]))
+				mac.Write([]byte(url))
+				digest := base64.URLEncoding.EncodeToString(mac.Sum(nil))
+				if options[0] == "" {
+					return proxyURL + "s" + digest + "/" + url
+				}
+				return proxyURL + options[0] + ",s" + digest + "/" + url
+			}
+			return proxyURL + options[0] + "/" + url
+		}
+
+		return url
+	}
+}
+
+func (a *App) ImageProxyRemover() (f func(string) string) {
+	proxyType, proxyURL, _, _ := a.imageProxyConfig()
+	if proxyType == "" {
+		return nil
+	}
+
+	return func(url string) string {
+		switch proxyType {
+		case "atmos/camo":
+			if strings.HasPrefix(url, proxyURL) {
+				if slash := strings.IndexByte(url[len(proxyURL):], '/'); slash >= 0 {
+					if decoded, err := hex.DecodeString(url[len(proxyURL)+slash+1:]); err == nil {
+						return string(decoded)
+					}
+				}
+			}
+		case "willnorris/imageproxy":
+			if strings.HasPrefix(url, proxyURL) {
+				if slash := strings.IndexByte(url[len(proxyURL):], '/'); slash >= 0 {
+					return url[len(proxyURL)+slash+1:]
+				}
+			}
+		}
+
+		return url
+	}
+}

app/post_test.go

@@ -185,3 +185,84 @@ func TestPostChannelMentions(t *testing.T) {
 		},
 	}, result.Props["channel_mentions"])
 }
+
+func TestImageProxy(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	for name, tc := range map[string]struct {
+		ProxyType       string
+		ProxyURL        string
+		ProxyOptions    string
+		ImageURL        string
+		ProxiedImageURL string
+	}{
+		"atmos/camo": {
+			ProxyType:       "atmos/camo",
+			ProxyURL:        "https://127.0.0.1",
+			ProxyOptions:    "foo",
+			ImageURL:        "http://mydomain.com/myimage",
+			ProxiedImageURL: "https://127.0.0.1/f8dace906d23689e8d5b12c3cefbedbf7b9b72f5/687474703a2f2f6d79646f6d61696e2e636f6d2f6d79696d616765",
+		},
+		"willnorris/imageproxy": {
+			ProxyType:       "willnorris/imageproxy",
+			ProxyURL:        "https://127.0.0.1",
+			ProxyOptions:    "x1000",
+			ImageURL:        "http://mydomain.com/myimage",
+			ProxiedImageURL: "https://127.0.0.1/x1000/http://mydomain.com/myimage",
+		},
+		"willnorris/imageproxy_WithSigning": {
+			ProxyType:       "willnorris/imageproxy",
+			ProxyURL:        "https://127.0.0.1",
+			ProxyOptions:    "x1000|foo",
+			ImageURL:        "http://mydomain.com/myimage",
+			ProxiedImageURL: "https://127.0.0.1/x1000,sbhHVoG5d60UvnNtGh6Iy6x4PaMmnsh8JfZ7JfErKjGU=/http://mydomain.com/myimage",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			th.App.UpdateConfig(func(cfg *model.Config) {
+				cfg.ServiceSettings.ImageProxyType = model.NewString(tc.ProxyType)
+				cfg.ServiceSettings.ImageProxyOptions = model.NewString(tc.ProxyOptions)
+				cfg.ServiceSettings.ImageProxyURL = model.NewString(tc.ProxyURL)
+			})
+
+			post := &model.Post{
+				Id:      model.NewId(),
+				Message: "![foo](" + tc.ImageURL + ")",
+			}
+
+			list := model.NewPostList()
+			list.Posts[post.Id] = post
+
+			assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostListWithProxyAddedToImageURLs(list).Posts[post.Id].Message)
+			assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostWithProxyAddedToImageURLs(post).Message)
+
+			assert.Equal(t, "![foo]("+tc.ImageURL+")", th.App.PostWithProxyRemovedFromImageURLs(post).Message)
+			post.Message = "![foo](" + tc.ProxiedImageURL + ")"
+			assert.Equal(t, "![foo]("+tc.ImageURL+")", th.App.PostWithProxyRemovedFromImageURLs(post).Message)
+		})
+	}
+}
+
+var imageProxyBenchmarkSink *model.Post
+
+func BenchmarkPostWithProxyRemovedFromImageURLs(b *testing.B) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		cfg.ServiceSettings.ImageProxyType = model.NewString("willnorris/imageproxy")
+		cfg.ServiceSettings.ImageProxyOptions = model.NewString("x1000|foo")
+		cfg.ServiceSettings.ImageProxyURL = model.NewString("https://127.0.0.1")
+	})
+
+	post := &model.Post{
+		Message: "![foo](http://mydomain.com/myimage)",
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		imageProxyBenchmarkSink = th.App.PostWithProxyAddedToImageURLs(post)
+	}
+}

app/reaction.go

@@ -62,6 +62,6 @@ func (a *App) sendReactionEvent(event string, reaction *model.Reaction, post *mo
 	post.HasReactions = true
 	post.UpdateAt = model.GetMillis()
 	umessage := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
-	umessage.Add("post", post.ToJson())
+	umessage.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
 	a.Publish(umessage)
 }

config/default.json

@@ -56,7 +56,9 @@
         "EnablePreviewFeatures": true,
         "CloseUnusedDirectMessages": false,
         "EnableTutorial": true,
-        "ExperimentalEnableDefaultChannelLeaveJoinMessages": true
+        "ExperimentalEnableDefaultChannelLeaveJoinMessages": true,
+        "ImageProxyType": "",
+        "ImageProxyURL": ""
     },
     "TeamSettings": {
         "SiteName": "Mattermost",

i18n/en.json

@@ -4814,6 +4814,14 @@
     "id": "model.config.is_valid.listen_address.app_error",
     "translation": "Invalid listen address for service settings Must be set."
   },
+  {
+    "id": "model.config.is_valid.image_proxy_type.app_error",
+    "translation": "Invalid image proxy type for service settings."
+  },
+  {
+    "id": "model.config.is_valid.atmos_camo_image_proxy_options.app_error",
+    "translation": "Invalid atmos/camo image proxy options for service settings. Must be set to your shared key."
+  },
   {
     "id": "model.config.is_valid.localization.available_locales.app_error",
     "translation": "Available Languages must contain Default Client Language"

model/config.go

@@ -214,6 +214,9 @@ type ServiceSettings struct {
 	EnablePreviewFeatures                             *bool
 	EnableTutorial                                    *bool
 	ExperimentalEnableDefaultChannelLeaveJoinMessages *bool
+	ImageProxyType                                    *string
+	ImageProxyURL                                     *string
+	ImageProxyOptions                                 *string
 }
 
 func (s *ServiceSettings) SetDefaults() {
@@ -250,7 +253,7 @@ func (s *ServiceSettings) SetDefaults() {
 	}
 
 	if s.AllowedUntrustedInternalConnections == nil {
-		s.AllowedUntrustedInternalConnections = new(string)
+		s.AllowedUntrustedInternalConnections = NewString("")
 	}
 
 	if s.EnableMultifactorAuthentication == nil {
@@ -418,6 +421,18 @@ func (s *ServiceSettings) SetDefaults() {
 	if s.ExperimentalEnableDefaultChannelLeaveJoinMessages == nil {
 		s.ExperimentalEnableDefaultChannelLeaveJoinMessages = NewBool(true)
 	}
+
+	if s.ImageProxyType == nil {
+		s.ImageProxyType = NewString("")
+	}
+
+	if s.ImageProxyURL == nil {
+		s.ImageProxyURL = NewString("")
+	}
+
+	if s.ImageProxyOptions == nil {
+		s.ImageProxyOptions = NewString("")
+	}
 }
 
 type ClusterSettings struct {
@@ -2050,6 +2065,16 @@ func (ss *ServiceSettings) isValid() *AppError {
 		return NewAppError("Config.IsValid", "model.config.is_valid.listen_address.app_error", nil, "", http.StatusBadRequest)
 	}
 
+	switch *ss.ImageProxyType {
+	case "", "willnorris/imageproxy":
+	case "atmos/camo":
+		if *ss.ImageProxyOptions == "" {
+			return NewAppError("Config.IsValid", "model.config.is_valid.atmos_camo_image_proxy_options.app_error", nil, "", http.StatusBadRequest)
+		}
+	default:
+		return NewAppError("Config.IsValid", "model.config.is_valid.image_proxy_type.app_error", nil, "", http.StatusBadRequest)
+	}
+
 	return nil
 }
 

model/post.go

@@ -8,8 +8,11 @@ import (
 	"io"
 	"net/http"
 	"regexp"
+	"sort"
 	"strings"
 	"unicode/utf8"
+
+	"github.com/mattermost/mattermost-server/utils/markdown"
 )
 
 const (
@@ -43,18 +46,25 @@ const (
 )
 
 type Post struct {
-	Id            string          `json:"id"`
-	CreateAt      int64           `json:"create_at"`
-	UpdateAt      int64           `json:"update_at"`
-	EditAt        int64           `json:"edit_at"`
-	DeleteAt      int64           `json:"delete_at"`
-	IsPinned      bool            `json:"is_pinned"`
-	UserId        string          `json:"user_id"`
-	ChannelId     string          `json:"channel_id"`
-	RootId        string          `json:"root_id"`
-	ParentId      string          `json:"parent_id"`
-	OriginalId    string          `json:"original_id"`
-	Message       string          `json:"message"`
+	Id         string `json:"id"`
+	CreateAt   int64  `json:"create_at"`
+	UpdateAt   int64  `json:"update_at"`
+	EditAt     int64  `json:"edit_at"`
+	DeleteAt   int64  `json:"delete_at"`
+	IsPinned   bool   `json:"is_pinned"`
+	UserId     string `json:"user_id"`
+	ChannelId  string `json:"channel_id"`
+	RootId     string `json:"root_id"`
+	ParentId   string `json:"parent_id"`
+	OriginalId string `json:"original_id"`
+
+	Message string `json:"message"`
+
+	// MessageSource will contain the message as submitted by the user if Message has been modified
+	// by Mattermost for presentation (e.g if an image proxy is being used). It should be used to
+	// populate edit boxes if present.
+	MessageSource string `json:"message_source,omitempty" db:"-"`
+
 	Type          string          `json:"type"`
 	Props         StringInterface `json:"props"`
 	Hashtags      string          `json:"hashtags"`
@@ -72,6 +82,14 @@ type PostPatch struct {
 	HasReactions *bool            `json:"has_reactions"`
 }
 
+func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch {
+	copy := *o
+	if copy.Message != nil {
+		*copy.Message = RewriteImageURLs(*o.Message, f)
+	}
+	return &copy
+}
+
 type PostForIndexing struct {
 	Post
 	TeamId         string `json:"team_id"`
@@ -392,3 +410,87 @@ func (o *Post) GenerateActionIds() {
 		}
 	}
 }
+
+var markdownDestinationEscaper = strings.NewReplacer(
+	`\`, `\\`,
+	`<`, `\<`,
+	`>`, `\>`,
+	`(`, `\(`,
+	`)`, `\)`,
+)
+
+// WithRewrittenImageURLs returns a new shallow copy of the post where the message has been
+// rewritten via RewriteImageURLs.
+func (o *Post) WithRewrittenImageURLs(f func(string) string) *Post {
+	copy := *o
+	copy.Message = RewriteImageURLs(o.Message, f)
+	if copy.MessageSource == "" && copy.Message != o.Message {
+		copy.MessageSource = o.Message
+	}
+	return &copy
+}
+
+// RewriteImageURLs takes a message and returns a copy that has all of the image URLs replaced
+// according to the function f. For each image URL, f will be invoked, and the resulting markdown
+// will contain the URL returned by that invocation instead.
+//
+// Image URLs are destination URLs used in inline images or reference definitions that are used
+// anywhere in the input markdown as an image.
+func RewriteImageURLs(message string, f func(string) string) string {
+	if !strings.Contains(message, "![") {
+		return message
+	}
+
+	var ranges []markdown.Range
+
+	markdown.Inspect(message, func(blockOrInline interface{}) bool {
+		switch v := blockOrInline.(type) {
+		case *markdown.ReferenceImage:
+			ranges = append(ranges, v.ReferenceDefinition.RawDestination)
+		case *markdown.InlineImage:
+			ranges = append(ranges, v.RawDestination)
+		default:
+			return true
+		}
+		return true
+	})
+
+	if ranges == nil {
+		return message
+	}
+
+	sort.Slice(ranges, func(i, j int) bool {
+		return ranges[i].Position < ranges[j].Position
+	})
+
+	copyRanges := make([]markdown.Range, 0, len(ranges))
+	urls := make([]string, 0, len(ranges))
+	resultLength := len(message)
+
+	start := 0
+	for i, r := range ranges {
+		switch {
+		case i == 0:
+		case r.Position != ranges[i-1].Position:
+			start = ranges[i-1].End
+		default:
+			continue
+		}
+		original := message[r.Position:r.End]
+		replacement := markdownDestinationEscaper.Replace(f(markdown.Unescape(original)))
+		resultLength += len(replacement) - len(original)
+		copyRanges = append(copyRanges, markdown.Range{Position: start, End: r.Position})
+		urls = append(urls, replacement)
+	}
+
+	result := make([]byte, resultLength)
+
+	offset := 0
+	for i, r := range copyRanges {
+		offset += copy(result[offset:], message[r.Position:r.End])
+		offset += copy(result[offset:], urls[i])
+	}
+	copy(result[offset:], message[ranges[len(ranges)-1].End:])
+
+	return string(result)
+}

model/post_list.go

@@ -21,6 +21,15 @@ func NewPostList() *PostList {
 	}
 }
 
+func (o *PostList) WithRewrittenImageURLs(f func(string) string) *PostList {
+	copy := *o
+	copy.Posts = make(map[string]*Post)
+	for id, post := range o.Posts {
+		copy.Posts[id] = post.WithRewrittenImageURLs(f)
+	}
+	return &copy
+}
+
 func (o *PostList) StripActionIntegrations() {
 	posts := o.Posts
 	o.Posts = make(map[string]*Post)

model/post_test.go

@@ -4,6 +4,7 @@
 package model
 
 import (
+	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -173,3 +174,129 @@ func TestPostSanitizeProps(t *testing.T) {
 		t.Fatal("should not be nil")
 	}
 }
+
+var markdownSample, markdownSampleWithRewrittenImageURLs string
+
+func init() {
+	bytes, err := ioutil.ReadFile("testdata/markdown-sample.md")
+	if err != nil {
+		panic(err)
+	}
+	markdownSample = string(bytes)
+
+	bytes, err = ioutil.ReadFile("testdata/markdown-sample-with-rewritten-image-urls.md")
+	if err != nil {
+		panic(err)
+	}
+	markdownSampleWithRewrittenImageURLs = string(bytes)
+}
+
+func TestRewriteImageURLs(t *testing.T) {
+	for name, tc := range map[string]struct {
+		Markdown string
+		Expected string
+	}{
+		"Empty": {
+			Markdown: ``,
+			Expected: ``,
+		},
+		"NoImages": {
+			Markdown: `foo`,
+			Expected: `foo`,
+		},
+		"Link": {
+			Markdown: `[foo](/url)`,
+			Expected: `[foo](/url)`,
+		},
+		"Image": {
+			Markdown: `![foo](/url)`,
+			Expected: `![foo](rewritten:/url)`,
+		},
+		"SpacedURL": {
+			Markdown: `![foo]( /url )`,
+			Expected: `![foo]( rewritten:/url )`,
+		},
+		"Title": {
+			Markdown: `![foo](/url "title")`,
+			Expected: `![foo](rewritten:/url "title")`,
+		},
+		"Parentheses": {
+			Markdown: `![foo](/url(1) "title")`,
+			Expected: `![foo](rewritten:/url\(1\) "title")`,
+		},
+		"AngleBrackets": {
+			Markdown: `![foo](</url\<1\>\\> "title")`,
+			Expected: `![foo](<rewritten:/url\<1\>\\> "title")`,
+		},
+		"MultipleLines": {
+			Markdown: `![foo](
+				</url\<1\>\\>
+				"title"
+			)`,
+			Expected: `![foo](
+				<rewritten:/url\<1\>\\>
+				"title"
+			)`,
+		},
+		"ReferenceLink": {
+			Markdown: `[foo]: </url\<1\>\\> "title"
+		 		[foo]`,
+			Expected: `[foo]: </url\<1\>\\> "title"
+		 		[foo]`,
+		},
+		"ReferenceImage": {
+			Markdown: `[foo]: </url\<1\>\\> "title"
+		 		![foo]`,
+			Expected: `[foo]: <rewritten:/url\<1\>\\> "title"
+		 		![foo]`,
+		},
+		"MultipleReferenceImages": {
+			Markdown: `[foo]: </url1> "title"
+				[bar]: </url2>
+				[baz]: /url3 "title"
+				[qux]: /url4
+				![foo]![qux]`,
+			Expected: `[foo]: <rewritten:/url1> "title"
+				[bar]: </url2>
+				[baz]: /url3 "title"
+				[qux]: rewritten:/url4
+				![foo]![qux]`,
+		},
+		"DuplicateReferences": {
+			Markdown: `[foo]: </url1> "title"
+				[foo]: </url2>
+				[foo]: /url3 "title"
+				[foo]: /url4
+				![foo]![foo]![foo]`,
+			Expected: `[foo]: <rewritten:/url1> "title"
+				[foo]: </url2>
+				[foo]: /url3 "title"
+				[foo]: /url4
+				![foo]![foo]![foo]`,
+		},
+		"TrailingURL": {
+			Markdown: "![foo]\n\n[foo]: /url",
+			Expected: "![foo]\n\n[foo]: rewritten:/url",
+		},
+		"Sample": {
+			Markdown: markdownSample,
+			Expected: markdownSampleWithRewrittenImageURLs,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, tc.Expected, RewriteImageURLs(tc.Markdown, func(url string) string {
+				return "rewritten:" + url
+			}))
+		})
+	}
+}
+
+var rewriteImageURLsSink string
+
+func BenchmarkRewriteImageURLs(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		rewriteImageURLsSink = RewriteImageURLs(markdownSample, func(url string) string {
+			return "rewritten:" + url
+		})
+	}
+}

model/testdata/markdown-sample-with-rewritten-image-urls.md

@@ -0,0 +1,245 @@
+---
+__Advertisement :)__
+
+- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image
+  resize in browser.
+- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly
+  i18n with plurals support and easy syntax.
+
+You will like those projects!
+
+---
+
+# h1 Heading 8-)
+## h2 Heading
+### h3 Heading
+#### h4 Heading
+##### h5 Heading
+###### h6 Heading
+
+
+## Horizontal Rules
+
+___
+
+---
+
+***
+
+
+## Typographic replacements
+
+Enable typographer option to see result.
+
+(c) (C) (r) (R) (tm) (TM) (p) (P) +-
+
+test.. test... test..... test?..... test!....
+
+!!!!!! ???? ,,  -- ---
+
+"Smartypants, double quotes" and 'single quotes'
+
+
+## Emphasis
+
+**This is bold text**
+
+__This is bold text__
+
+*This is italic text*
+
+_This is italic text_
+
+~~Strikethrough~~
+
+
+## Blockquotes
+
+
+> Blockquotes can also be nested...
+>> ...by using additional greater-than signs right next to each other...
+> > > ...or with spaces between arrows.
+
+
+## Lists
+
+Unordered
+
++ Create a list by starting a line with `+`, `-`, or `*`
++ Sub-lists are made by indenting 2 spaces:
+  - Marker character change forces new list start:
+    * Ac tristique libero volutpat at
+    + Facilisis in pretium nisl aliquet
+    - Nulla volutpat aliquam velit
++ Very easy!
+
+Ordered
+
+1. Lorem ipsum dolor sit amet
+2. Consectetur adipiscing elit
+3. Integer molestie lorem at massa
+
+
+1. You can use sequential numbers...
+1. ...or keep all the numbers as `1.`
+
+Start numbering with offset:
+
+57. foo
+1. bar
+
+
+## Code
+
+Inline `code`
+
+Indented code
+
+    // Some comments
+    line 1 of code
+    line 2 of code
+    line 3 of code
+
+
+Block code "fences"
+
+```
+Sample text here...
+```
+
+Syntax highlighting
+
+``` js
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+## Tables
+
+| Option | Description |
+| ------ | ----------- |
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+Right aligned columns
+
+| Option | Description |
+| ------:| -----------:|
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+
+## Links
+
+[link text](http://dev.nodeca.com)
+
+[link with title](http://nodeca.github.io/pica/demo/ "title text!")
+
+Autoconverted link https://github.com/nodeca/pica (enable linkify to see)
+
+
+## Images
+
+![Minion](rewritten:https://octodex.github.com/images/minion.png)
+![Stormtroopocat](rewritten:https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat")
+
+Like links, Images also have a footnote style syntax
+
+![Alt text][id]
+
+With a reference later in the document defining the URL location:
+
+[id]: rewritten:https://octodex.github.com/images/dojocat.jpg  "The Dojocat"
+
+
+## Plugins
+
+The killer feature of `markdown-it` is very effective support of
+[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin).
+
+
+### [Emojies](https://github.com/markdown-it/markdown-it-emoji)
+
+> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum:
+>
+> Shortcuts (emoticons): :-) :-( 8-) ;)
+
+see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji.
+
+
+### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup)
+
+- 19^th^
+- H~2~O
+
+
+### [\<ins>](https://github.com/markdown-it/markdown-it-ins)
+
+++Inserted text++
+
+
+### [\<mark>](https://github.com/markdown-it/markdown-it-mark)
+
+==Marked text==
+
+
+### [Footnotes](https://github.com/markdown-it/markdown-it-footnote)
+
+Footnote 1 link[^first].
+
+Footnote 2 link[^second].
+
+Inline footnote^[Text of inline footnote] definition.
+
+Duplicated footnote reference[^second].
+
+[^first]: Footnote **can have markup**
+
+    and multiple paragraphs.
+
+[^second]: Footnote text.
+
+
+### [Definition lists](https://github.com/markdown-it/markdown-it-deflist)
+
+Term 1
+
+:   Definition 1
+with lazy continuation.
+
+Term 2 with *inline markup*
+
+:   Definition 2
+
+        { some code, part of Definition 2 }
+
+    Third paragraph of definition 2.
+
+_Compact style:_
+
+Term 1
+  ~ Definition 1
+
+Term 2
+  ~ Definition 2a
+  ~ Definition 2b
+
+
+### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr)
+
+This is HTML abbreviation example.
+
+It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on.
+
+*[HTML]: Hyper Text Markup Language
+
+### [Custom containers](https://github.com/markdown-it/markdown-it-container)
+
+::: warning
+*here be dragons*
+:::

model/testdata/markdown-sample.md

@@ -0,0 +1,245 @@
+---
+__Advertisement :)__
+
+- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image
+  resize in browser.
+- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly
+  i18n with plurals support and easy syntax.
+
+You will like those projects!
+
+---
+
+# h1 Heading 8-)
+## h2 Heading
+### h3 Heading
+#### h4 Heading
+##### h5 Heading
+###### h6 Heading
+
+
+## Horizontal Rules
+
+___
+
+---
+
+***
+
+
+## Typographic replacements
+
+Enable typographer option to see result.
+
+(c) (C) (r) (R) (tm) (TM) (p) (P) +-
+
+test.. test... test..... test?..... test!....
+
+!!!!!! ???? ,,  -- ---
+
+"Smartypants, double quotes" and 'single quotes'
+
+
+## Emphasis
+
+**This is bold text**
+
+__This is bold text__
+
+*This is italic text*
+
+_This is italic text_
+
+~~Strikethrough~~
+
+
+## Blockquotes
+
+
+> Blockquotes can also be nested...
+>> ...by using additional greater-than signs right next to each other...
+> > > ...or with spaces between arrows.
+
+
+## Lists
+
+Unordered
+
++ Create a list by starting a line with `+`, `-`, or `*`
++ Sub-lists are made by indenting 2 spaces:
+  - Marker character change forces new list start:
+    * Ac tristique libero volutpat at
+    + Facilisis in pretium nisl aliquet
+    - Nulla volutpat aliquam velit
++ Very easy!
+
+Ordered
+
+1. Lorem ipsum dolor sit amet
+2. Consectetur adipiscing elit
+3. Integer molestie lorem at massa
+
+
+1. You can use sequential numbers...
+1. ...or keep all the numbers as `1.`
+
+Start numbering with offset:
+
+57. foo
+1. bar
+
+
+## Code
+
+Inline `code`
+
+Indented code
+
+    // Some comments
+    line 1 of code
+    line 2 of code
+    line 3 of code
+
+
+Block code "fences"
+
+```
+Sample text here...
+```
+
+Syntax highlighting
+
+``` js
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+## Tables
+
+| Option | Description |
+| ------ | ----------- |
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+Right aligned columns
+
+| Option | Description |
+| ------:| -----------:|
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+
+## Links
+
+[link text](http://dev.nodeca.com)
+
+[link with title](http://nodeca.github.io/pica/demo/ "title text!")
+
+Autoconverted link https://github.com/nodeca/pica (enable linkify to see)
+
+
+## Images
+
+![Minion](https://octodex.github.com/images/minion.png)
+![Stormtroopocat](https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat")
+
+Like links, Images also have a footnote style syntax
+
+![Alt text][id]
+
+With a reference later in the document defining the URL location:
+
+[id]: https://octodex.github.com/images/dojocat.jpg  "The Dojocat"
+
+
+## Plugins
+
+The killer feature of `markdown-it` is very effective support of
+[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin).
+
+
+### [Emojies](https://github.com/markdown-it/markdown-it-emoji)
+
+> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum:
+>
+> Shortcuts (emoticons): :-) :-( 8-) ;)
+
+see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji.
+
+
+### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup)
+
+- 19^th^
+- H~2~O
+
+
+### [\<ins>](https://github.com/markdown-it/markdown-it-ins)
+
+++Inserted text++
+
+
+### [\<mark>](https://github.com/markdown-it/markdown-it-mark)
+
+==Marked text==
+
+
+### [Footnotes](https://github.com/markdown-it/markdown-it-footnote)
+
+Footnote 1 link[^first].
+
+Footnote 2 link[^second].
+
+Inline footnote^[Text of inline footnote] definition.
+
+Duplicated footnote reference[^second].
+
+[^first]: Footnote **can have markup**
+
+    and multiple paragraphs.
+
+[^second]: Footnote text.
+
+
+### [Definition lists](https://github.com/markdown-it/markdown-it-deflist)
+
+Term 1
+
+:   Definition 1
+with lazy continuation.
+
+Term 2 with *inline markup*
+
+:   Definition 2
+
+        { some code, part of Definition 2 }
+
+    Third paragraph of definition 2.
+
+_Compact style:_
+
+Term 1
+  ~ Definition 1
+
+Term 2
+  ~ Definition 2a
+  ~ Definition 2b
+
+
+### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr)
+
+This is HTML abbreviation example.
+
+It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on.
+
+*[HTML]: Hyper Text Markup Language
+
+### [Custom containers](https://github.com/markdown-it/markdown-it-container)
+
+::: warning
+*here be dragons*
+:::

utils/markdown/block_quote.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+type BlockQuote struct {
+	blockBase
+	markdown string
+
+	Children []Block
+}
+
+func (b *BlockQuote) Continuation(indentation int, r Range) *continuation {
+	if indentation > 3 {
+		return nil
+	}
+	s := b.markdown[r.Position:r.End]
+	if s == "" || s[0] != '>' {
+		return nil
+	}
+	remaining := Range{r.Position + 1, r.End}
+	indentation, indentationBytes := countIndentation(b.markdown, remaining)
+	if indentation > 0 {
+		indentation--
+	}
+	return &continuation{
+		Indentation: indentation,
+		Remaining:   Range{remaining.Position + indentationBytes, remaining.End},
+	}
+}
+
+func (b *BlockQuote) AddChild(openBlocks []Block) []Block {
+	b.Children = append(b.Children, openBlocks[0])
+	return openBlocks
+}
+
+func blockQuoteStart(markdown string, indent int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
+	if indent > 3 {
+		return nil
+	}
+	s := markdown[r.Position:r.End]
+	if s == "" || s[0] != '>' {
+		return nil
+	}
+
+	block := &BlockQuote{
+		markdown: markdown,
+	}
+	r.Position++
+	if len(s) > 1 && s[1] == ' ' {
+		r.Position++
+	}
+
+	indent, bytes := countIndentation(markdown, r)
+
+	ret := []Block{block}
+	if descendants := blockStartOrParagraph(markdown, indent, Range{r.Position + bytes, r.End}, nil, nil); descendants != nil {
+		block.Children = append(block.Children, descendants[0])
+		ret = append(ret, descendants...)
+	}
+	return ret
+}

utils/markdown/blocks.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"strings"
+)
+
+type continuation struct {
+	Indentation int
+	Remaining   Range
+}
+
+type Block interface {
+	Continuation(indentation int, r Range) *continuation
+	AddLine(indentation int, r Range) bool
+	Close()
+	AllowsBlockStarts() bool
+	HasTrailingBlankLine() bool
+}
+
+type blockBase struct{}
+
+func (*blockBase) AddLine(indentation int, r Range) bool { return false }
+func (*blockBase) Close()                                {}
+func (*blockBase) AllowsBlockStarts() bool               { return true }
+func (*blockBase) HasTrailingBlankLine() bool            { return false }
+
+type ContainerBlock interface {
+	Block
+	AddChild(openBlocks []Block) []Block
+}
+
+type Range struct {
+	Position int
+	End      int
+}
+
+func closeBlocks(blocks []Block, referenceDefinitions *[]*ReferenceDefinition) {
+	for _, block := range blocks {
+		block.Close()
+		if p, ok := block.(*Paragraph); ok && len(p.ReferenceDefinitions) > 0 {
+			*referenceDefinitions = append(*referenceDefinitions, p.ReferenceDefinitions...)
+		}
+	}
+}
+
+func ParseBlocks(markdown string, lines []Line) (*Document, []*ReferenceDefinition) {
+	document := &Document{}
+	var referenceDefinitions []*ReferenceDefinition
+
+	openBlocks := []Block{document}
+
+	for _, line := range lines {
+		r := line.Range
+		lastMatchIndex := 0
+
+		indentation, indentationBytes := countIndentation(markdown, r)
+		r = Range{r.Position + indentationBytes, r.End}
+
+		for i, block := range openBlocks {
+			if continuation := block.Continuation(indentation, r); continuation != nil {
+				indentation = continuation.Indentation
+				r = continuation.Remaining
+				additionalIndentation, additionalIndentationBytes := countIndentation(markdown, r)
+				r = Range{r.Position + additionalIndentationBytes, r.End}
+				indentation += additionalIndentation
+				lastMatchIndex = i
+			} else {
+				break
+			}
+		}
+
+		if openBlocks[lastMatchIndex].AllowsBlockStarts() {
+			if newBlocks := blockStart(markdown, indentation, r, openBlocks[:lastMatchIndex+1], openBlocks[lastMatchIndex+1:]); newBlocks != nil {
+				didAdd := false
+				for i := lastMatchIndex; i >= 0; i-- {
+					if container, ok := openBlocks[i].(ContainerBlock); ok {
+						if newBlocks := container.AddChild(newBlocks); newBlocks != nil {
+							closeBlocks(openBlocks[i+1:], &referenceDefinitions)
+							openBlocks = openBlocks[:i+1]
+							openBlocks = append(openBlocks, newBlocks...)
+							didAdd = true
+							break
+						}
+					}
+				}
+				if didAdd {
+					continue
+				}
+			}
+		}
+
+		isBlank := strings.TrimSpace(markdown[r.Position:r.End]) == ""
+		if paragraph, ok := openBlocks[len(openBlocks)-1].(*Paragraph); ok && !isBlank {
+			paragraph.Text = append(paragraph.Text, r)
+			continue
+		}
+
+		closeBlocks(openBlocks[lastMatchIndex+1:], &referenceDefinitions)
+		openBlocks = openBlocks[:lastMatchIndex+1]
+
+		if openBlocks[lastMatchIndex].AddLine(indentation, r) {
+			continue
+		}
+
+		if paragraph := newParagraph(markdown, r); paragraph != nil {
+			for i := lastMatchIndex; i >= 0; i-- {
+				if container, ok := openBlocks[i].(ContainerBlock); ok {
+					if newBlocks := container.AddChild([]Block{paragraph}); newBlocks != nil {
+						closeBlocks(openBlocks[i+1:], &referenceDefinitions)
+						openBlocks = openBlocks[:i+1]
+						openBlocks = append(openBlocks, newBlocks...)
+						break
+					}
+				}
+			}
+		}
+	}
+
+	closeBlocks(openBlocks, &referenceDefinitions)
+
+	return document, referenceDefinitions
+}
+
+func blockStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
+	if r.Position >= r.End {
+		return nil
+	}
+
+	if start := blockQuoteStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
+		return start
+	} else if start := listStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
+		return start
+	} else if start := indentedCodeStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
+		return start
+	} else if start := fencedCodeStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
+		return start
+	}
+
+	return nil
+}
+
+func blockStartOrParagraph(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
+	if start := blockStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil {
+		return start
+	}
+	if paragraph := newParagraph(markdown, r); paragraph != nil {
+		return []Block{paragraph}
+	}
+	return nil
+}

utils/markdown/document.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+type Document struct {
+	blockBase
+
+	Children []Block
+}
+
+func (b *Document) Continuation(indentation int, r Range) *continuation {
+	return &continuation{
+		Indentation: indentation,
+		Remaining:   r,
+	}
+}
+
+func (b *Document) AddChild(openBlocks []Block) []Block {
+	b.Children = append(b.Children, openBlocks[0])
+	return openBlocks
+}

utils/markdown/fenced_code.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"strings"
+)
+
+type FencedCodeLine struct {
+	Indentation int
+	Range       Range
+}
+
+type FencedCode struct {
+	blockBase
+	markdown           string
+	didSeeClosingFence bool
+
+	Indentation  int
+	OpeningFence Range
+	RawInfo      Range
+	RawCode      []FencedCodeLine
+}
+
+func (b *FencedCode) Code() (result string) {
+	for _, code := range b.RawCode {
+		result += strings.Repeat(" ", code.Indentation) + b.markdown[code.Range.Position:code.Range.End]
+	}
+	return
+}
+
+func (b *FencedCode) Info() string {
+	return Unescape(b.markdown[b.RawInfo.Position:b.RawInfo.End])
+}
+
+func (b *FencedCode) Continuation(indentation int, r Range) *continuation {
+	if b.didSeeClosingFence {
+		return nil
+	}
+	return &continuation{
+		Indentation: indentation,
+		Remaining:   r,
+	}
+}
+
+func (b *FencedCode) AddLine(indentation int, r Range) bool {
+	s := b.markdown[r.Position:r.End]
+	if indentation <= 3 && strings.HasPrefix(s, b.markdown[b.OpeningFence.Position:b.OpeningFence.End]) {
+		suffix := strings.TrimSpace(s[b.OpeningFence.End-b.OpeningFence.Position:])
+		isClosingFence := true
+		for _, c := range suffix {
+			if c != rune(s[0]) {
+				isClosingFence = false
+				break
+			}
+		}
+		if isClosingFence {
+			b.didSeeClosingFence = true
+			return true
+		}
+	}
+
+	if indentation >= b.Indentation {
+		indentation -= b.Indentation
+	} else {
+		indentation = 0
+	}
+
+	b.RawCode = append(b.RawCode, FencedCodeLine{
+		Indentation: indentation,
+		Range:       r,
+	})
+	return true
+}
+
+func (b *FencedCode) AllowsBlockStarts() bool {
+	return false
+}
+
+func fencedCodeStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
+	s := markdown[r.Position:r.End]
+
+	if !strings.HasPrefix(s, "```") && !strings.HasPrefix(s, "~~~") {
+		return nil
+	}
+
+	fenceCharacter := rune(s[0])
+	fenceLength := 3
+	for _, c := range s[3:] {
+		if c == fenceCharacter {
+			fenceLength++
+		} else {
+			break
+		}
+	}
+
+	for i := r.Position + fenceLength; i < r.End; i++ {
+		if markdown[i] == '`' {
+			return nil
+		}
+	}
+
+	return []Block{
+		&FencedCode{
+			markdown:     markdown,
+			Indentation:  indentation,
+			RawInfo:      trimRightSpace(markdown, Range{r.Position + fenceLength, r.End}),
+			OpeningFence: Range{r.Position, r.Position + fenceLength},
+		},
+	}
+}

utils/markdown/html.go

@@ -0,0 +1,186 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"fmt"
+	"strings"
+)
+
+var htmlEscaper = strings.NewReplacer(
+	`&`, "&",
+	`<`, "&lt;",
+	`>`, "&gt;",
+	`"`, "&quot;",
+)
+
+// RenderHTML produces HTML with the same behavior as the example renderer used in the CommonMark
+// reference materials except for one slight difference: for brevity, no unnecessary whitespace is
+// inserted between elements. The output is not defined by the CommonMark spec, and it exists
+// primarily as an aid in testing.
+func RenderHTML(markdown string) string {
+	return RenderBlockHTML(Parse(markdown))
+}
+
+func RenderBlockHTML(block Block, referenceDefinitions []*ReferenceDefinition) (result string) {
+	return renderBlockHTML(block, referenceDefinitions, false)
+}
+
+func renderBlockHTML(block Block, referenceDefinitions []*ReferenceDefinition, isTightList bool) (result string) {
+	switch v := block.(type) {
+	case *Document:
+		for _, block := range v.Children {
+			result += RenderBlockHTML(block, referenceDefinitions)
+		}
+	case *Paragraph:
+		if len(v.Text) == 0 {
+			return
+		}
+		if !isTightList {
+			result += "<p>"
+		}
+		for _, inline := range v.ParseInlines(referenceDefinitions) {
+			result += RenderInlineHTML(inline)
+		}
+		if !isTightList {
+			result += "</p>"
+		}
+	case *List:
+		if v.IsOrdered {
+			if v.OrderedStart != 1 {
+				result += fmt.Sprintf(`<ol start="%v">`, v.OrderedStart)
+			} else {
+				result += "<ol>"
+			}
+		} else {
+			result += "<ul>"
+		}
+		for _, block := range v.Children {
+			result += renderBlockHTML(block, referenceDefinitions, !v.IsLoose)
+		}
+		if v.IsOrdered {
+			result += "</ol>"
+		} else {
+			result += "</ul>"
+		}
+	case *ListItem:
+		result += "<li>"
+		for _, block := range v.Children {
+			result += renderBlockHTML(block, referenceDefinitions, isTightList)
+		}
+		result += "</li>"
+	case *BlockQuote:
+		result += "<blockquote>"
+		for _, block := range v.Children {
+			result += RenderBlockHTML(block, referenceDefinitions)
+		}
+		result += "</blockquote>"
+	case *FencedCode:
+		if info := v.Info(); info != "" {
+			language := strings.Fields(info)[0]
+			result += `<pre><code class="language-` + htmlEscaper.Replace(language) + `">`
+		} else {
+			result += "<pre><code>"
+		}
+		result += htmlEscaper.Replace(v.Code()) + "</code></pre>"
+	case *IndentedCode:
+		result += "<pre><code>" + htmlEscaper.Replace(v.Code()) + "</code></pre>"
+	default:
+		panic(fmt.Sprintf("missing case for type %T", v))
+	}
+	return
+}
+
+func escapeURL(url string) (result string) {
+	for i := 0; i < len(url); {
+		switch b := url[i]; b {
+		case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '-', '_', '.', '!', '~', '*', '\'', '(', ')', '#':
+			result += string(b)
+			i++
+		default:
+			if b == '%' && i+2 < len(url) && isHexByte(url[i+1]) && isHexByte(url[i+2]) {
+				result += url[i : i+3]
+				i += 3
+			} else if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') {
+				result += string(b)
+				i++
+			} else {
+				result += fmt.Sprintf("%%%0X", b)
+				i++
+			}
+		}
+	}
+	return
+}
+
+func RenderInlineHTML(inline Inline) (result string) {
+	switch v := inline.(type) {
+	case *Text:
+		return htmlEscaper.Replace(v.Text)
+	case *HardLineBreak:
+		return "<br />"
+	case *SoftLineBreak:
+		return "\n"
+	case *CodeSpan:
+		return "<code>" + htmlEscaper.Replace(v.Code) + "</code>"
+	case *InlineImage:
+		result += `<img src="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `" alt="` + htmlEscaper.Replace(renderImageAltText(v.Children)) + `"`
+		if title := v.Title(); title != "" {
+			result += ` title="` + htmlEscaper.Replace(title) + `"`
+		}
+		result += ` />`
+	case *ReferenceImage:
+		result += `<img src="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `" alt="` + htmlEscaper.Replace(renderImageAltText(v.Children)) + `"`
+		if title := v.Title(); title != "" {
+			result += ` title="` + htmlEscaper.Replace(title) + `"`
+		}
+		result += ` />`
+	case *InlineLink:
+		result += `<a href="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `"`
+		if title := v.Title(); title != "" {
+			result += ` title="` + htmlEscaper.Replace(title) + `"`
+		}
+		result += `>`
+		for _, inline := range v.Children {
+			result += RenderInlineHTML(inline)
+		}
+		result += "</a>"
+	case *ReferenceLink:
+		result += `<a href="` + htmlEscaper.Replace(escapeURL(v.Destination())) + `"`
+		if title := v.Title(); title != "" {
+			result += ` title="` + htmlEscaper.Replace(title) + `"`
+		}
+		result += `>`
+		for _, inline := range v.Children {
+			result += RenderInlineHTML(inline)
+		}
+		result += "</a>"
+	default:
+		panic(fmt.Sprintf("missing case for type %T", v))
+	}
+	return
+}
+
+func renderImageAltText(children []Inline) (result string) {
+	for _, inline := range children {
+		result += renderImageChildAltText(inline)
+	}
+	return
+}
+
+func renderImageChildAltText(inline Inline) (result string) {
+	switch v := inline.(type) {
+	case *Text:
+		return v.Text
+	case *InlineImage:
+		for _, inline := range v.Children {
+			result += renderImageChildAltText(inline)
+		}
+	case *InlineLink:
+		for _, inline := range v.Children {
+			result += renderImageChildAltText(inline)
+		}
+	}
+	return
+}

utils/markdown/indented_code.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"strings"
+)
+
+type IndentedCodeLine struct {
+	Indentation int
+	Range       Range
+}
+
+type IndentedCode struct {
+	blockBase
+	markdown string
+
+	RawCode []IndentedCodeLine
+}
+
+func (b *IndentedCode) Code() (result string) {
+	for _, code := range b.RawCode {
+		result += strings.Repeat(" ", code.Indentation) + b.markdown[code.Range.Position:code.Range.End]
+	}
+	return
+}
+
+func (b *IndentedCode) Continuation(indentation int, r Range) *continuation {
+	if indentation >= 4 {
+		return &continuation{
+			Indentation: indentation - 4,
+			Remaining:   r,
+		}
+	}
+	s := b.markdown[r.Position:r.End]
+	if strings.TrimSpace(s) == "" {
+		return &continuation{
+			Remaining: r,
+		}
+	}
+	return nil
+}
+
+func (b *IndentedCode) AddLine(indentation int, r Range) bool {
+	b.RawCode = append(b.RawCode, IndentedCodeLine{
+		Indentation: indentation,
+		Range:       r,
+	})
+	return true
+}
+
+func (b *IndentedCode) Close() {
+	for {
+		last := b.RawCode[len(b.RawCode)-1]
+		s := b.markdown[last.Range.Position:last.Range.End]
+		if strings.TrimRight(s, "\r\n") == "" {
+			b.RawCode = b.RawCode[:len(b.RawCode)-1]
+		} else {
+			break
+		}
+	}
+}
+
+func (b *IndentedCode) AllowsBlockStarts() bool {
+	return false
+}
+
+func indentedCodeStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
+	if len(unmatchedBlocks) > 0 {
+		if _, ok := unmatchedBlocks[len(unmatchedBlocks)-1].(*Paragraph); ok {
+			return nil
+		}
+	} else if len(matchedBlocks) > 0 {
+		if _, ok := matchedBlocks[len(matchedBlocks)-1].(*Paragraph); ok {
+			return nil
+		}
+	}
+
+	if indentation < 4 {
+		return nil
+	}
+
+	s := markdown[r.Position:r.End]
+	if strings.TrimSpace(s) == "" {
+		return nil
+	}
+
+	return []Block{
+		&IndentedCode{
+			markdown: markdown,
+			RawCode: []IndentedCodeLine{{
+				Indentation: indentation - 4,
+				Range:       r,
+			}},
+		},
+	}
+}

utils/markdown/inlines.go

@@ -0,0 +1,489 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"container/list"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type Inline interface {
+	IsInline() bool
+}
+
+type inlineBase struct{}
+
+func (inlineBase) IsInline() bool { return true }
+
+type Text struct {
+	inlineBase
+
+	Text string
+}
+
+type CodeSpan struct {
+	inlineBase
+
+	Code string
+}
+
+type HardLineBreak struct {
+	inlineBase
+}
+
+type SoftLineBreak struct {
+	inlineBase
+}
+
+type InlineLinkOrImage struct {
+	inlineBase
+
+	Children []Inline
+
+	RawDestination Range
+
+	markdown string
+	rawTitle string
+}
+
+func (i *InlineLinkOrImage) Destination() string {
+	return Unescape(i.markdown[i.RawDestination.Position:i.RawDestination.End])
+}
+
+func (i *InlineLinkOrImage) Title() string {
+	return Unescape(i.rawTitle)
+}
+
+type InlineLink struct {
+	InlineLinkOrImage
+}
+
+type InlineImage struct {
+	InlineLinkOrImage
+}
+
+type ReferenceLinkOrImage struct {
+	inlineBase
+	*ReferenceDefinition
+
+	Children []Inline
+}
+
+type ReferenceLink struct {
+	ReferenceLinkOrImage
+}
+
+type ReferenceImage struct {
+	ReferenceLinkOrImage
+}
+
+type delimiterType int
+
+const (
+	linkOpeningDelimiter delimiterType = iota
+	imageOpeningDelimiter
+)
+
+type delimiter struct {
+	Type       delimiterType
+	IsInactive bool
+	TextNode   int
+	Range      Range
+}
+
+type inlineParser struct {
+	markdown             string
+	ranges               []Range
+	referenceDefinitions []*ReferenceDefinition
+
+	raw            string
+	position       int
+	inlines        []Inline
+	delimiterStack *list.List
+}
+
+func newInlineParser(markdown string, ranges []Range, referenceDefinitions []*ReferenceDefinition) *inlineParser {
+	return &inlineParser{
+		markdown:             markdown,
+		ranges:               ranges,
+		referenceDefinitions: referenceDefinitions,
+		delimiterStack:       list.New(),
+	}
+}
+
+func (p *inlineParser) parseBackticks() {
+	count := 1
+	for i := p.position + 1; i < len(p.raw) && p.raw[i] == '`'; i++ {
+		count++
+	}
+	opening := p.raw[p.position : p.position+count]
+	search := p.position + count
+	for search < len(p.raw) {
+		end := strings.Index(p.raw[search:], opening)
+		if end == -1 {
+			break
+		}
+		if search+end+count < len(p.raw) && p.raw[search+end+count] == '`' {
+			search += end + count
+			for search < len(p.raw) && p.raw[search] == '`' {
+				search++
+			}
+			continue
+		}
+		code := strings.Join(strings.Fields(p.raw[p.position+count:search+end]), " ")
+		p.position = search + end + count
+		p.inlines = append(p.inlines, &CodeSpan{
+			Code: code,
+		})
+		return
+	}
+	p.position += len(opening)
+	p.inlines = append(p.inlines, &Text{
+		Text: opening,
+	})
+}
+
+func (p *inlineParser) parseLineEnding() {
+	if p.position >= 1 && p.raw[p.position-1] == '\t' {
+		p.inlines = append(p.inlines, &HardLineBreak{})
+	} else if p.position >= 2 && p.raw[p.position-1] == ' ' && (p.raw[p.position-2] == '\t' || p.raw[p.position-1] == ' ') {
+		p.inlines = append(p.inlines, &HardLineBreak{})
+	} else {
+		p.inlines = append(p.inlines, &SoftLineBreak{})
+	}
+	p.position++
+	if p.position < len(p.raw) && p.raw[p.position] == '\n' {
+		p.position++
+	}
+}
+
+func (p *inlineParser) parseEscapeCharacter() {
+	if p.position+1 < len(p.raw) && isEscapableByte(p.raw[p.position+1]) {
+		p.inlines = append(p.inlines, &Text{
+			Text: string(p.raw[p.position+1]),
+		})
+		p.position += 2
+	} else {
+		p.inlines = append(p.inlines, &Text{
+			Text: `\`,
+		})
+		p.position++
+	}
+}
+
+func (p *inlineParser) parseText() {
+	if next := strings.IndexAny(p.raw[p.position:], "\r\n\\`&![]"); next == -1 {
+		p.inlines = append(p.inlines, &Text{
+			Text: strings.TrimRightFunc(p.raw[p.position:], isWhitespace),
+		})
+		p.position = len(p.raw)
+	} else {
+		if p.raw[p.position+next] == '\r' || p.raw[p.position+next] == '\n' {
+			p.inlines = append(p.inlines, &Text{
+				Text: strings.TrimRightFunc(p.raw[p.position:p.position+next], isWhitespace),
+			})
+		} else {
+			p.inlines = append(p.inlines, &Text{
+				Text: p.raw[p.position : p.position+next],
+			})
+		}
+		p.position += next
+	}
+}
+
+func (p *inlineParser) parseLinkOrImageDelimiter() {
+	if p.raw[p.position] == '[' {
+		p.inlines = append(p.inlines, &Text{
+			Text: "[",
+		})
+		p.delimiterStack.PushBack(&delimiter{
+			Type:     linkOpeningDelimiter,
+			TextNode: len(p.inlines) - 1,
+			Range:    Range{p.position, p.position + 1},
+		})
+		p.position++
+	} else if p.raw[p.position] == '!' && p.position+1 < len(p.raw) && p.raw[p.position+1] == '[' {
+		p.inlines = append(p.inlines, &Text{
+			Text: "![",
+		})
+		p.delimiterStack.PushBack(&delimiter{
+			Type:     imageOpeningDelimiter,
+			TextNode: len(p.inlines) - 1,
+			Range:    Range{p.position, p.position + 2},
+		})
+		p.position += 2
+	} else {
+		p.inlines = append(p.inlines, &Text{
+			Text: "!",
+		})
+		p.position++
+	}
+}
+
+func (p *inlineParser) peekAtInlineLinkDestinationAndTitle(position int) (destination, title Range, end int, ok bool) {
+	if position >= len(p.raw) || p.raw[position] != '(' {
+		return
+	}
+	position++
+
+	destinationStart := nextNonWhitespace(p.raw, position)
+	if destinationStart >= len(p.raw) {
+		return
+	} else if p.raw[destinationStart] == ')' {
+		return Range{destinationStart, destinationStart}, Range{destinationStart, destinationStart}, destinationStart + 1, true
+	}
+
+	destination, end, ok = parseLinkDestination(p.raw, destinationStart)
+	if !ok {
+		return
+	}
+	position = end
+
+	if position < len(p.raw) && isWhitespaceByte(p.raw[position]) {
+		titleStart := nextNonWhitespace(p.raw, position)
+		if titleStart >= len(p.raw) {
+			return
+		} else if p.raw[titleStart] == ')' {
+			return destination, Range{titleStart, titleStart}, titleStart + 1, true
+		}
+
+		title, end, ok = parseLinkTitle(p.raw, titleStart)
+		if !ok {
+			return
+		}
+		position = end
+	}
+
+	closingPosition := nextNonWhitespace(p.raw, position)
+	if closingPosition >= len(p.raw) || p.raw[closingPosition] != ')' {
+		return Range{}, Range{}, 0, false
+	}
+
+	return destination, title, closingPosition + 1, true
+}
+
+func (p *inlineParser) referenceDefinition(label string) *ReferenceDefinition {
+	clean := strings.Join(strings.Fields(label), " ")
+	for _, d := range p.referenceDefinitions {
+		if strings.EqualFold(clean, strings.Join(strings.Fields(d.Label()), " ")) {
+			return d
+		}
+	}
+	return nil
+}
+
+func (p *inlineParser) lookForLinkOrImage() {
+	for element := p.delimiterStack.Back(); element != nil; element = element.Prev() {
+		d := element.Value.(*delimiter)
+		if d.Type != imageOpeningDelimiter && d.Type != linkOpeningDelimiter {
+			continue
+		}
+		if d.IsInactive {
+			p.delimiterStack.Remove(element)
+			break
+		}
+
+		var inline Inline
+
+		if destination, title, next, ok := p.peekAtInlineLinkDestinationAndTitle(p.position + 1); ok {
+			destinationMarkdownPosition := relativeToAbsolutePosition(p.ranges, destination.Position)
+			linkOrImage := InlineLinkOrImage{
+				Children:       append([]Inline(nil), p.inlines[d.TextNode+1:]...),
+				RawDestination: Range{destinationMarkdownPosition, destinationMarkdownPosition + destination.End - destination.Position},
+				markdown:       p.markdown,
+				rawTitle:       p.raw[title.Position:title.End],
+			}
+			if d.Type == imageOpeningDelimiter {
+				inline = &InlineImage{linkOrImage}
+			} else {
+				inline = &InlineLink{linkOrImage}
+			}
+			p.position = next
+		} else {
+			referenceLabel := ""
+			label, next, hasLinkLabel := parseLinkLabel(p.raw, p.position+1)
+			if hasLinkLabel && label.End > label.Position {
+				referenceLabel = p.raw[label.Position:label.End]
+			} else {
+				referenceLabel = p.raw[d.Range.End:p.position]
+				if !hasLinkLabel {
+					next = p.position + 1
+				}
+			}
+			if referenceLabel != "" {
+				if reference := p.referenceDefinition(referenceLabel); reference != nil {
+					linkOrImage := ReferenceLinkOrImage{
+						ReferenceDefinition: reference,
+						Children:            append([]Inline(nil), p.inlines[d.TextNode+1:]...),
+					}
+					if d.Type == imageOpeningDelimiter {
+						inline = &ReferenceImage{linkOrImage}
+					} else {
+						inline = &ReferenceLink{linkOrImage}
+					}
+					p.position = next
+				}
+			}
+		}
+
+		if inline != nil {
+			if d.Type == imageOpeningDelimiter {
+				p.inlines = append(p.inlines[:d.TextNode], inline)
+			} else {
+				p.inlines = append(p.inlines[:d.TextNode], inline)
+				for element := element.Prev(); element != nil; element = element.Prev() {
+					if d := element.Value.(*delimiter); d.Type == linkOpeningDelimiter {
+						d.IsInactive = true
+					}
+				}
+			}
+			p.delimiterStack.Remove(element)
+			return
+		} else {
+			p.delimiterStack.Remove(element)
+			break
+		}
+	}
+	p.inlines = append(p.inlines, &Text{
+		Text: "]",
+	})
+	p.position++
+}
+
+func CharacterReference(ref string) string {
+	if ref == "" {
+		return ""
+	}
+	if ref[0] == '#' {
+		if len(ref) < 2 {
+			return ""
+		}
+		n := 0
+		if ref[1] == 'X' || ref[1] == 'x' {
+			if len(ref) < 3 {
+				return ""
+			}
+			for i := 2; i < len(ref); i++ {
+				if i > 9 {
+					return ""
+				}
+				d := ref[i]
+				switch {
+				case d >= '0' && d <= '9':
+					n = n*16 + int(d-'0')
+				case d >= 'a' && d <= 'f':
+					n = n*16 + 10 + int(d-'a')
+				case d >= 'A' && d <= 'F':
+					n = n*16 + 10 + int(d-'A')
+				default:
+					return ""
+				}
+			}
+		} else {
+			for i := 1; i < len(ref); i++ {
+				if i > 8 || ref[i] < '0' || ref[i] > '9' {
+					return ""
+				}
+				n = n*10 + int(ref[i]-'0')
+			}
+		}
+		c := rune(n)
+		if c == '\u0000' || !utf8.ValidRune(c) {
+			return string(unicode.ReplacementChar)
+		}
+		return string(c)
+	}
+	if entity, ok := htmlEntities[ref]; ok {
+		return entity
+	}
+	return ""
+}
+
+func (p *inlineParser) parseCharacterReference() {
+	p.position++
+	if semicolon := strings.IndexByte(p.raw[p.position:], ';'); semicolon == -1 {
+		p.inlines = append(p.inlines, &Text{
+			Text: "&",
+		})
+	} else if s := CharacterReference(p.raw[p.position : p.position+semicolon]); s != "" {
+		p.position += semicolon + 1
+		p.inlines = append(p.inlines, &Text{
+			Text: s,
+		})
+	} else {
+		p.inlines = append(p.inlines, &Text{
+			Text: "&",
+		})
+	}
+}
+
+func (p *inlineParser) Parse() []Inline {
+	for _, r := range p.ranges {
+		p.raw += p.markdown[r.Position:r.End]
+	}
+
+	for p.position < len(p.raw) {
+		c, _ := utf8.DecodeRuneInString(p.raw[p.position:])
+
+		switch c {
+		case '\r', '\n':
+			p.parseLineEnding()
+		case '\\':
+			p.parseEscapeCharacter()
+		case '`':
+			p.parseBackticks()
+		case '&':
+			p.parseCharacterReference()
+		case '!', '[':
+			p.parseLinkOrImageDelimiter()
+		case ']':
+			p.lookForLinkOrImage()
+		default:
+			p.parseText()
+		}
+	}
+
+	return p.inlines
+}
+
+func ParseInlines(markdown string, ranges []Range, referenceDefinitions []*ReferenceDefinition) (inlines []Inline) {
+	return newInlineParser(markdown, ranges, referenceDefinitions).Parse()
+}
+
+func Unescape(markdown string) string {
+	ret := ""
+
+	position := 0
+	for position < len(markdown) {
+		c, cSize := utf8.DecodeRuneInString(markdown[position:])
+
+		switch c {
+		case '\\':
+			if position+1 < len(markdown) && isEscapableByte(markdown[position+1]) {
+				ret += string(markdown[position+1])
+				position += 2
+			} else {
+				ret += `\`
+				position++
+			}
+		case '&':
+			position++
+			if semicolon := strings.IndexByte(markdown[position:], ';'); semicolon == -1 {
+				ret += "&"
+			} else if s := CharacterReference(markdown[position : position+semicolon]); s != "" {
+				position += semicolon + 1
+				ret += s
+			} else {
+				ret += "&"
+			}
+		default:
+			ret += string(c)
+			position += cSize
+		}
+	}
+
+	return ret
+}

utils/markdown/inspect.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+// Inspect traverses the markdown tree in depth-first order. If f returns true, Inspect invokes f
+// recursively for each child of the block or inline, followed by a call of f(nil).
+func Inspect(markdown string, f func(interface{}) bool) {
+	document, referenceDefinitions := Parse(markdown)
+	InspectBlock(document, func(block Block) bool {
+		if !f(block) {
+			return false
+		}
+		switch v := block.(type) {
+		case *Paragraph:
+			for _, inline := range v.ParseInlines(referenceDefinitions) {
+				InspectInline(inline, func(inline Inline) bool {
+					return f(inline)
+				})
+			}
+		}
+		return true
+	})
+}
+
+// InspectBlock traverses the blocks in depth-first order, starting with block. If f returns true,
+// InspectBlock invokes f recursively for each child of the block, followed by a call of f(nil).
+func InspectBlock(block Block, f func(Block) bool) {
+	if !f(block) {
+		return
+	}
+	switch v := block.(type) {
+	case *Document:
+		for _, child := range v.Children {
+			InspectBlock(child, f)
+		}
+	case *List:
+		for _, child := range v.Children {
+			InspectBlock(child, f)
+		}
+	case *ListItem:
+		for _, child := range v.Children {
+			InspectBlock(child, f)
+		}
+	case *BlockQuote:
+		for _, child := range v.Children {
+			InspectBlock(child, f)
+		}
+	}
+	f(nil)
+}
+
+// InspectInline traverses the blocks in depth-first order, starting with block. If f returns true,
+// InspectInline invokes f recursively for each child of the block, followed by a call of f(nil).
+func InspectInline(inline Inline, f func(Inline) bool) {
+	if !f(inline) {
+		return
+	}
+	switch v := inline.(type) {
+	case *InlineImage:
+		for _, child := range v.Children {
+			InspectInline(child, f)
+		}
+	case *InlineLink:
+		for _, child := range v.Children {
+			InspectInline(child, f)
+		}
+	case *ReferenceImage:
+		for _, child := range v.Children {
+			InspectInline(child, f)
+		}
+	case *ReferenceLink:
+		for _, child := range v.Children {
+			InspectInline(child, f)
+		}
+	}
+	f(nil)
+}

utils/markdown/inspect_test.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInspect(t *testing.T) {
+	markdown := `
+[foo]: bar
+- a
+  > [![]()]()
+  > [![foo]][foo]
+- d
+`
+
+	visited := []string{}
+	level := 0
+	Inspect(markdown, func(blockOrInline interface{}) bool {
+		if blockOrInline == nil {
+			level--
+		} else {
+			visited = append(visited, strings.Repeat(" ", level*4)+strings.TrimPrefix(fmt.Sprintf("%T", blockOrInline), "*markdown."))
+			level++
+		}
+		return true
+	})
+
+	assert.Equal(t, []string{
+		"Document",
+		"    Paragraph",
+		"    List",
+		"        ListItem",
+		"            Paragraph",
+		"                Text",
+		"            BlockQuote",
+		"                Paragraph",
+		"                    InlineLink",
+		"                        InlineImage",
+		"                    SoftLineBreak",
+		"                    ReferenceLink",
+		"                        ReferenceImage",
+		"                            Text",
+		"        ListItem",
+		"            Paragraph",
+		"                Text",
+	}, visited)
+}

utils/markdown/lines.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+type Line struct {
+	Range
+}
+
+func ParseLines(markdown string) (lines []Line) {
+	lineStartPosition := 0
+	isAfterCarriageReturn := false
+	for position, r := range markdown {
+		if r == '\n' {
+			lines = append(lines, Line{Range{lineStartPosition, position + 1}})
+			lineStartPosition = position + 1
+		} else if isAfterCarriageReturn {
+			lines = append(lines, Line{Range{lineStartPosition, position}})
+			lineStartPosition = position
+		}
+		isAfterCarriageReturn = r == '\r'
+	}
+	if lineStartPosition < len(markdown) {
+		lines = append(lines, Line{Range{lineStartPosition, len(markdown)}})
+	}
+	return
+}

utils/markdown/lines_test.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseLines(t *testing.T) {
+	assert.Equal(t, []Line{
+		{Range{0, 4}}, {Range{4, 7}},
+	}, ParseLines("foo\nbar"))
+
+	assert.Equal(t, []Line{
+		{Range{0, 5}}, {Range{5, 8}},
+	}, ParseLines("foo\r\nbar"))
+
+	assert.Equal(t, []Line{
+		{Range{0, 4}}, {Range{4, 6}}, {Range{6, 9}},
+	}, ParseLines("foo\r\r\nbar"))
+
+	assert.Equal(t, []Line{
+		{Range{0, 4}},
+	}, ParseLines("foo\n"))
+
+	assert.Equal(t, []Line{
+		{Range{0, 4}},
+	}, ParseLines("foo\r"))
+
+	assert.Equal(t, []Line{
+		{Range{0, 5}},
+	}, ParseLines("foo\r\n"))
+}

utils/markdown/links.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"unicode/utf8"
+)
+
+func parseLinkDestination(markdown string, position int) (raw Range, next int, ok bool) {
+	if position >= len(markdown) {
+		return
+	}
+
+	if markdown[position] == '<' {
+		isEscaped := false
+
+		for offset, c := range []byte(markdown[position+1:]) {
+			if isEscaped {
+				isEscaped = false
+				if isEscapableByte(c) {
+					continue
+				}
+			}
+
+			if c == '\\' {
+				isEscaped = true
+			} else if c == '<' {
+				break
+			} else if c == '>' {
+				return Range{position + 1, position + 1 + offset}, position + 1 + offset + 1, true
+			} else if isWhitespaceByte(c) {
+				break
+			}
+		}
+	}
+
+	openCount := 0
+	isEscaped := false
+	for offset, c := range []byte(markdown[position:]) {
+		if isEscaped {
+			isEscaped = false
+			if isEscapableByte(c) {
+				continue
+			}
+		}
+
+		switch c {
+		case '\\':
+			isEscaped = true
+		case '(':
+			openCount++
+		case ')':
+			if openCount < 1 {
+				return Range{position, position + offset}, position + offset, true
+			}
+			openCount--
+		default:
+			if isWhitespaceByte(c) {
+				return Range{position, position + offset}, position + offset, true
+			}
+		}
+	}
+	return Range{position, len(markdown)}, len(markdown), true
+}
+
+func parseLinkTitle(markdown string, position int) (raw Range, next int, ok bool) {
+	if position >= len(markdown) {
+		return
+	}
+
+	originalPosition := position
+
+	var closer byte
+	switch markdown[position] {
+	case '"', '\'':
+		closer = markdown[position]
+	case '(':
+		closer = ')'
+	default:
+		return
+	}
+	position++
+
+	for position < len(markdown) {
+		switch markdown[position] {
+		case '\\':
+			position++
+			if position < len(markdown) && isEscapableByte(markdown[position]) {
+				position++
+			}
+		case closer:
+			return Range{originalPosition + 1, position}, position + 1, true
+		default:
+			position++
+		}
+	}
+
+	return
+}
+
+func parseLinkLabel(markdown string, position int) (raw Range, next int, ok bool) {
+	if position >= len(markdown) || markdown[position] != '[' {
+		return
+	}
+
+	originalPosition := position
+	position++
+
+	for position < len(markdown) {
+		switch markdown[position] {
+		case '\\':
+			position++
+			if position < len(markdown) && isEscapableByte(markdown[position]) {
+				position++
+			}
+		case '[':
+			return
+		case ']':
+			if position-originalPosition >= 1000 && utf8.RuneCountInString(markdown[originalPosition:position]) >= 1000 {
+				return
+			}
+			return Range{originalPosition + 1, position}, position + 1, true
+		default:
+			position++
+		}
+	}
+
+	return
+}

utils/markdown/list.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"strings"
+)
+
+type ListItem struct {
+	blockBase
+	markdown                    string
+	hasTrailingBlankLine        bool
+	hasBlankLineBetweenChildren bool
+
+	Indentation int
+	Children    []Block
+}
+
+func (b *ListItem) Continuation(indentation int, r Range) *continuation {
+	s := b.markdown[r.Position:r.End]
+	if strings.TrimSpace(s) == "" {
+		if b.Children == nil {
+			return nil
+		}
+		return &continuation{
+			Remaining: r,
+		}
+	}
+	if indentation < b.Indentation {
+		return nil
+	}
+	return &continuation{
+		Indentation: indentation - b.Indentation,
+		Remaining:   r,
+	}
+}
+
+func (b *ListItem) AddChild(openBlocks []Block) []Block {
+	b.Children = append(b.Children, openBlocks[0])
+	if b.hasTrailingBlankLine {
+		b.hasBlankLineBetweenChildren = true
+	}
+	b.hasTrailingBlankLine = false
+	return openBlocks
+}
+
+func (b *ListItem) AddLine(indentation int, r Range) bool {
+	isBlank := strings.TrimSpace(b.markdown[r.Position:r.End]) == ""
+	if isBlank {
+		b.hasTrailingBlankLine = true
+	}
+	return false
+}
+
+func (b *ListItem) HasTrailingBlankLine() bool {
+	return b.hasTrailingBlankLine || (len(b.Children) > 0 && b.Children[len(b.Children)-1].HasTrailingBlankLine())
+}
+
+func (b *ListItem) isLoose() bool {
+	if b.hasBlankLineBetweenChildren {
+		return true
+	}
+	for i, child := range b.Children {
+		if i < len(b.Children)-1 && child.HasTrailingBlankLine() {
+			return true
+		}
+	}
+	return false
+}
+
+type List struct {
+	blockBase
+	markdown                    string
+	hasTrailingBlankLine        bool
+	hasBlankLineBetweenChildren bool
+
+	IsLoose           bool
+	IsOrdered         bool
+	OrderedStart      int
+	BulletOrDelimiter byte
+	Children          []*ListItem
+}
+
+func (b *List) Continuation(indentation int, r Range) *continuation {
+	s := b.markdown[r.Position:r.End]
+	if strings.TrimSpace(s) == "" {
+		return &continuation{
+			Remaining: r,
+		}
+	}
+	return &continuation{
+		Indentation: indentation,
+		Remaining:   r,
+	}
+}
+
+func (b *List) AddChild(openBlocks []Block) []Block {
+	if item, ok := openBlocks[0].(*ListItem); ok {
+		b.Children = append(b.Children, item)
+		if b.hasTrailingBlankLine {
+			b.hasBlankLineBetweenChildren = true
+		}
+		b.hasTrailingBlankLine = false
+		return openBlocks
+	} else if list, ok := openBlocks[0].(*List); ok {
+		if len(list.Children) == 1 && list.IsOrdered == b.IsOrdered && list.BulletOrDelimiter == b.BulletOrDelimiter {
+			return b.AddChild(openBlocks[1:])
+		}
+	}
+	return nil
+}
+
+func (b *List) AddLine(indentation int, r Range) bool {
+	isBlank := strings.TrimSpace(b.markdown[r.Position:r.End]) == ""
+	if isBlank {
+		b.hasTrailingBlankLine = true
+	}
+	return false
+}
+
+func (b *List) HasTrailingBlankLine() bool {
+	return b.hasTrailingBlankLine || (len(b.Children) > 0 && b.Children[len(b.Children)-1].HasTrailingBlankLine())
+}
+
+func (b *List) isLoose() bool {
+	if b.hasBlankLineBetweenChildren {
+		return true
+	}
+	for i, child := range b.Children {
+		if child.isLoose() || (i < len(b.Children)-1 && child.HasTrailingBlankLine()) {
+			return true
+		}
+	}
+	return false
+}
+
+func (b *List) Close() {
+	b.IsLoose = b.isLoose()
+}
+
+func parseListMarker(markdown string, r Range) (success, isOrdered bool, orderedStart int, bulletOrDelimiter byte, markerWidth int, remaining Range) {
+	digits := 0
+	n := 0
+	for i := r.Position; i < r.End && markdown[i] >= '0' && markdown[i] <= '9'; i++ {
+		digits++
+		n = n*10 + int(markdown[i]-'0')
+	}
+	if digits > 0 {
+		if digits > 9 || r.Position+digits >= r.End {
+			return
+		}
+		next := markdown[r.Position+digits]
+		if next != '.' && next != ')' {
+			return
+		}
+		return true, true, n, next, digits + 1, Range{r.Position + digits + 1, r.End}
+	}
+	if r.Position >= r.End {
+		return
+	}
+	next := markdown[r.Position]
+	if next != '-' && next != '+' && next != '*' {
+		return
+	}
+	return true, false, 0, next, 1, Range{r.Position + 1, r.End}
+}
+
+func listStart(markdown string, indent int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block {
+	afterList := false
+	if len(matchedBlocks) > 0 {
+		_, afterList = matchedBlocks[len(matchedBlocks)-1].(*List)
+	}
+	if !afterList && indent > 3 {
+		return nil
+	}
+
+	success, isOrdered, orderedStart, bulletOrDelimiter, markerWidth, remaining := parseListMarker(markdown, r)
+	if !success {
+		return nil
+	}
+
+	isBlank := strings.TrimSpace(markdown[remaining.Position:remaining.End]) == ""
+	if len(matchedBlocks) > 0 && len(unmatchedBlocks) == 0 {
+		if _, ok := matchedBlocks[len(matchedBlocks)-1].(*Paragraph); ok {
+			if isBlank || (isOrdered && orderedStart != 1) {
+				return nil
+			}
+		}
+	}
+
+	indentAfterMarker, indentBytesAfterMarker := countIndentation(markdown, remaining)
+	if !isBlank && indentAfterMarker < 1 {
+		return nil
+	}
+
+	remaining = Range{remaining.Position + indentBytesAfterMarker, remaining.End}
+	consumedIndentAfterMarker := indentAfterMarker
+	if isBlank || indentAfterMarker >= 5 {
+		consumedIndentAfterMarker = 1
+	}
+
+	listItem := &ListItem{
+		markdown:    markdown,
+		Indentation: indent + markerWidth + consumedIndentAfterMarker,
+	}
+	list := &List{
+		markdown:          markdown,
+		IsOrdered:         isOrdered,
+		OrderedStart:      orderedStart,
+		BulletOrDelimiter: bulletOrDelimiter,
+		Children:          []*ListItem{listItem},
+	}
+	ret := []Block{list, listItem}
+	if descendants := blockStartOrParagraph(markdown, indentAfterMarker-consumedIndentAfterMarker, remaining, nil, nil); descendants != nil {
+		listItem.Children = append(listItem.Children, descendants[0])
+		ret = append(ret, descendants...)
+	}
+	return ret
+}

utils/markdown/markdown.go

@@ -0,0 +1,132 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+// This package implements a parser for the subset of the CommonMark spec necessary for us to do
+// server-side processing. It is not a full implementation and lacks many features. But it is
+// complete enough to efficiently and accurately allow us to do what we need to like rewrite image
+// URLs for proxying.
+package markdown
+
+import (
+	"strings"
+)
+
+func isEscapable(c rune) bool {
+	return c > ' ' && (c < '0' || (c > '9' && (c < 'A' || (c > 'Z' && (c < 'a' || (c > 'z' && c <= '~'))))))
+}
+
+func isEscapableByte(c byte) bool {
+	return isEscapable(rune(c))
+}
+
+func isWhitespace(c rune) bool {
+	switch c {
+	case ' ', '\t', '\n', '\u000b', '\u000c', '\r':
+		return true
+	default:
+		return false
+	}
+}
+
+func isWhitespaceByte(c byte) bool {
+	return isWhitespace(rune(c))
+}
+
+func isHex(c rune) bool {
+	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
+}
+
+func isHexByte(c byte) bool {
+	return isHex(rune(c))
+}
+
+func nextNonWhitespace(markdown string, position int) int {
+	for offset, c := range []byte(markdown[position:]) {
+		if !isWhitespaceByte(c) {
+			return position + offset
+		}
+	}
+	return len(markdown)
+}
+
+func nextLine(markdown string, position int) (linePosition int, skippedNonWhitespace bool) {
+	for i := position; i < len(markdown); i++ {
+		c := markdown[i]
+		if c == '\r' {
+			if i+1 < len(markdown) && markdown[i+1] == '\n' {
+				return i + 2, skippedNonWhitespace
+			}
+			return i + 1, skippedNonWhitespace
+		} else if c == '\n' {
+			return i + 1, skippedNonWhitespace
+		} else if !isWhitespaceByte(c) {
+			skippedNonWhitespace = true
+		}
+	}
+	return len(markdown), skippedNonWhitespace
+}
+
+func countIndentation(markdown string, r Range) (spaces, bytes int) {
+	for i := r.Position; i < r.End; i++ {
+		if markdown[i] == ' ' {
+			spaces++
+			bytes++
+		} else if markdown[i] == '\t' {
+			spaces += 4
+			bytes++
+		} else {
+			break
+		}
+	}
+	return
+}
+
+func trimLeftSpace(markdown string, r Range) Range {
+	s := markdown[r.Position:r.End]
+	trimmed := strings.TrimLeftFunc(s, isWhitespace)
+	return Range{r.Position, r.End - (len(s) - len(trimmed))}
+}
+
+func trimRightSpace(markdown string, r Range) Range {
+	s := markdown[r.Position:r.End]
+	trimmed := strings.TrimRightFunc(s, isWhitespace)
+	return Range{r.Position, r.End - (len(s) - len(trimmed))}
+}
+
+func relativeToAbsolutePosition(ranges []Range, position int) int {
+	rem := position
+	for _, r := range ranges {
+		l := r.End - r.Position
+		if rem < l {
+			return r.Position + rem
+		}
+		rem -= l
+	}
+	if len(ranges) == 0 {
+		return 0
+	}
+	return ranges[len(ranges)-1].End
+}
+
+func trimBytesFromRanges(ranges []Range, bytes int) (result []Range) {
+	rem := bytes
+	for _, r := range ranges {
+		if rem == 0 {
+			result = append(result, r)
+			continue
+		}
+		l := r.End - r.Position
+		if rem < l {
+			result = append(result, Range{r.Position + rem, r.End})
+			rem = 0
+			continue
+		}
+		rem -= l
+	}
+	return
+}
+
+func Parse(markdown string) (*Document, []*ReferenceDefinition) {
+	lines := ParseLines(markdown)
+	return ParseBlocks(markdown, lines)
+}

utils/markdown/paragraph.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+import (
+	"strings"
+)
+
+type Paragraph struct {
+	blockBase
+	markdown string
+
+	Text                 []Range
+	ReferenceDefinitions []*ReferenceDefinition
+}
+
+func (b *Paragraph) ParseInlines(referenceDefinitions []*ReferenceDefinition) []Inline {
+	return ParseInlines(b.markdown, b.Text, referenceDefinitions)
+}
+
+func (b *Paragraph) Continuation(indentation int, r Range) *continuation {
+	s := b.markdown[r.Position:r.End]
+	if strings.TrimSpace(s) == "" {
+		return nil
+	}
+	return &continuation{
+		Indentation: indentation,
+		Remaining:   r,
+	}
+}
+
+func (b *Paragraph) Close() {
+	for {
+		for i := 0; i < len(b.Text); i++ {
+			b.Text[i] = trimLeftSpace(b.markdown, b.Text[i])
+			if b.Text[i].Position < b.Text[i].End {
+				break
+			}
+		}
+
+		if len(b.Text) == 0 || b.Text[0].Position < b.Text[0].End && b.markdown[b.Text[0].Position] != '[' {
+			break
+		}
+
+		definition, remaining := parseReferenceDefinition(b.markdown, b.Text)
+		if definition == nil {
+			break
+		}
+		b.ReferenceDefinitions = append(b.ReferenceDefinitions, definition)
+		b.Text = remaining
+	}
+
+	for i := len(b.Text) - 1; i >= 0; i-- {
+		b.Text[i] = trimRightSpace(b.markdown, b.Text[i])
+		if b.Text[i].Position < b.Text[i].End {
+			break
+		}
+	}
+}
+
+func newParagraph(markdown string, r Range) *Paragraph {
+	s := markdown[r.Position:r.End]
+	if strings.TrimSpace(s) == "" {
+		return nil
+	}
+	return &Paragraph{
+		markdown: markdown,
+		Text:     []Range{r},
+	}
+}

utils/markdown/reference_definition.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package markdown
+
+type ReferenceDefinition struct {
+	RawDestination Range
+
+	markdown string
+	rawLabel string
+	rawTitle string
+}
+
+func (d *ReferenceDefinition) Destination() string {
+	return Unescape(d.markdown[d.RawDestination.Position:d.RawDestination.End])
+}
+
+func (d *ReferenceDefinition) Label() string {
+	return d.rawLabel
+}
+
+func (d *ReferenceDefinition) Title() string {
+	return Unescape(d.rawTitle)
+}
+
+func parseReferenceDefinition(markdown string, ranges []Range) (*ReferenceDefinition, []Range) {
+	raw := ""
+	for _, r := range ranges {
+		raw += markdown[r.Position:r.End]
+	}
+
+	label, next, ok := parseLinkLabel(raw, 0)
+	if !ok {
+		return nil, nil
+	}
+	position := next
+
+	if position >= len(raw) || raw[position] != ':' {
+		return nil, nil
+	}
+	position++
+
+	destination, next, ok := parseLinkDestination(raw, nextNonWhitespace(raw, position))
+	if !ok {
+		return nil, nil
+	}
+	position = next
+
+	absoluteDestination := relativeToAbsolutePosition(ranges, destination.Position)
+	ret := &ReferenceDefinition{
+		RawDestination: Range{absoluteDestination, absoluteDestination + destination.End - destination.Position},
+		markdown:       markdown,
+		rawLabel:       raw[label.Position:label.End],
+	}
+
+	if position < len(raw) && isWhitespaceByte(raw[position]) {
+		title, next, ok := parseLinkTitle(raw, nextNonWhitespace(raw, position))
+		if !ok {
+			if nextLine, skippedNonWhitespace := nextLine(raw, position); !skippedNonWhitespace {
+				return ret, trimBytesFromRanges(ranges, nextLine)
+			}
+			return nil, nil
+		}
+		if nextLine, skippedNonWhitespace := nextLine(raw, next); !skippedNonWhitespace {
+			ret.rawTitle = raw[title.Position:title.End]
+			return ret, trimBytesFromRanges(ranges, nextLine)
+		}
+	}
+
+	if nextLine, skippedNonWhitespace := nextLine(raw, position); !skippedNonWhitespace {
+		return ret, trimBytesFromRanges(ranges, nextLine)
+	}
+
+	return nil, nil
+}