Skip to content

Commit

Permalink
Add configuration to not pre-parse multipart form data
Browse files Browse the repository at this point in the history
  • Loading branch information
MarkMercAWS committed Apr 13, 2020
1 parent 402e095 commit fdedfa0
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 16 deletions.
28 changes: 15 additions & 13 deletions http.go
Original file line number Diff line number Diff line change
Expand Up @@ -937,10 +937,10 @@ func (req *Request) ReadLimitBody(r *bufio.Reader, maxBodySize int) error {
return err
}

return req.readLimitBody(r, maxBodySize, false)
return req.readLimitBody(r, maxBodySize, false, true)
}

func (req *Request) readLimitBody(r *bufio.Reader, maxBodySize int, getOnly bool) error {
func (req *Request) readLimitBody(r *bufio.Reader, maxBodySize int, getOnly bool, preParseMultipartForm bool) error {
// Do not reset the request here - the caller must reset it before
// calling this method.

Expand All @@ -955,7 +955,7 @@ func (req *Request) readLimitBody(r *bufio.Reader, maxBodySize int, getOnly bool
return nil
}

return req.ContinueReadBody(r, maxBodySize)
return req.ContinueReadBody(r, maxBodySize, preParseMultipartForm)
}

// MayContinue returns true if the request contains
Expand All @@ -979,24 +979,26 @@ func (req *Request) MayContinue() bool {
//
// If maxBodySize > 0 and the body size exceeds maxBodySize,
// then ErrBodyTooLarge is returned.
func (req *Request) ContinueReadBody(r *bufio.Reader, maxBodySize int) error {
func (req *Request) ContinueReadBody(r *bufio.Reader, maxBodySize int, preParseMultipartForm ...bool) error {
var err error
contentLength := req.Header.realContentLength()
if contentLength > 0 {
if maxBodySize > 0 && contentLength > maxBodySize {
return ErrBodyTooLarge
}

// Pre-read multipart form data of known length.
// This way we limit memory usage for large file uploads, since their contents
// is streamed into temporary files if file size exceeds defaultMaxInMemoryFileSize.
req.multipartFormBoundary = string(req.Header.MultipartFormBoundary())
if len(req.multipartFormBoundary) > 0 && len(req.Header.peek(strContentEncoding)) == 0 {
req.multipartForm, err = readMultipartForm(r, req.multipartFormBoundary, contentLength, defaultMaxInMemoryFileSize)
if err != nil {
req.Reset()
if len(preParseMultipartForm) == 0 || preParseMultipartForm[0] {
// Pre-read multipart form data of known length.
// This way we limit memory usage for large file uploads, since their contents
// is streamed into temporary files if file size exceeds defaultMaxInMemoryFileSize.
req.multipartFormBoundary = string(req.Header.MultipartFormBoundary())
if len(req.multipartFormBoundary) > 0 && len(req.Header.peek(strContentEncoding)) == 0 {
req.multipartForm, err = readMultipartForm(r, req.multipartFormBoundary, contentLength, defaultMaxInMemoryFileSize)
if err != nil {
req.Reset()
}
return err
}
return err
}
}

Expand Down
44 changes: 43 additions & 1 deletion http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -869,7 +869,7 @@ func TestRequestContinueReadBody(t *testing.T) {
t.Fatalf("MayContinue must return true")
}

if err := r.ContinueReadBody(br, 0); err != nil {
if err := r.ContinueReadBody(br, 0, true); err != nil {
t.Fatalf("error when reading request body: %s", err)
}
body := r.Body()
Expand All @@ -886,6 +886,48 @@ func TestRequestContinueReadBody(t *testing.T) {
}
}

func TestRequestContinueReadBodyDisablePrereadMultipartForm(t *testing.T) {
t.Parallel()

var w bytes.Buffer
mw := multipart.NewWriter(&w)
for i := 0; i < 10; i++ {
k := fmt.Sprintf("key_%d", i)
v := fmt.Sprintf("value_%d", i)
if err := mw.WriteField(k, v); err != nil {
t.Fatalf("unexpected error: %s", err)
}
}
boundary := mw.Boundary()
if err := mw.Close(); err != nil {
t.Fatalf("unexpected error: %s", err)
}
formData := w.Bytes()

s := fmt.Sprintf("POST / HTTP/1.1\r\nHost: aaa\r\nContent-Type: multipart/form-data; boundary=%s\r\nContent-Length: %d\r\n\r\n%s",
boundary, len(formData), formData)
br := bufio.NewReader(bytes.NewBufferString(s))

var r Request

if err := r.Header.Read(br); err != nil {
t.Fatalf("unexpected error reading headers: %s", err)
}

if err := r.readLimitBody(br, 10000, false, false); err != nil {
t.Fatalf("unexpected error reading body: %s", err)
}

if r.multipartForm != nil {
t.Fatalf("The multipartForm of the Request must be nil")
}

if string(formData) != string(r.Body()) {
t.Fatalf("The body given must equal the body in the Request")
}

}

func TestRequestMayContinue(t *testing.T) {
t.Parallel()

Expand Down
12 changes: 10 additions & 2 deletions server.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,14 @@ type Server struct {
// Server accepts all the requests by default.
GetOnly bool

// Will not pre parse Multipart Form data if set to true.
//
// This option is useful for servers that desire to treat
// multipart form data as a binary blob, or choose when to parse the data.
//
// Server pre parses multipart form data by default.
DisablePreParseMultipartForm bool

// Logs all errors, including the most frequent
// 'connection reset by peer', 'broken pipe' and 'connection timeout'
// errors. Such errors are common in production serving real-world
Expand Down Expand Up @@ -1988,7 +1996,7 @@ func (s *Server) serveConn(c net.Conn) (err error) {
}
}
//read body
err = ctx.Request.readLimitBody(br, maxRequestBodySize, s.GetOnly)
err = ctx.Request.readLimitBody(br, maxRequestBodySize, s.GetOnly, !s.DisablePreParseMultipartForm)
}
if err == nil {
// If we read any bytes off the wire, we're active.
Expand Down Expand Up @@ -2047,7 +2055,7 @@ func (s *Server) serveConn(c net.Conn) (err error) {
if br == nil {
br = acquireReader(ctx)
}
err = ctx.Request.ContinueReadBody(br, maxRequestBodySize)
err = ctx.Request.ContinueReadBody(br, maxRequestBodySize, !s.DisablePreParseMultipartForm)
if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil {
releaseReader(s, br)
br = nil
Expand Down

0 comments on commit fdedfa0

Please sign in to comment.