if someone logs a line longer than 65*1024 bytes before it contains a newline, the default ScanLines func in bufio.Scanner can't handle it. Copy that func and add a conditional to clear buffer and continue scanning in this situation.

This commit is contained in:
Megan Marsh 2019-11-08 12:34:36 -08:00
parent 643fce7b8a
commit 88530a7944
1 changed files with 39 additions and 0 deletions

39
log.go
View File

@ -2,6 +2,7 @@ package main
import ( import (
"bufio" "bufio"
"bytes"
"fmt" "fmt"
"io" "io"
"os" "os"
@ -30,6 +31,8 @@ func logOutput() (logOutput io.Writer, err error) {
// calls. // calls.
r, w := io.Pipe() r, w := io.Pipe()
scanner := bufio.NewScanner(r) scanner := bufio.NewScanner(r)
scanner.Split(ScanLinesSmallerThanBuffer)
go func(scanner *bufio.Scanner) { go func(scanner *bufio.Scanner) {
for scanner.Scan() { for scanner.Scan() {
if strings.Contains(scanner.Text(), "ui:") { if strings.Contains(scanner.Text(), "ui:") {
@ -47,3 +50,39 @@ func logOutput() (logOutput io.Writer, err error) {
return return
} }
// The below functions come from bufio.Scanner with a small tweak, to fix an
// edgecase where the default ScanFunc fails: sometimes, if someone tries to
// log a line that is longer than 64*1024 bytes long before it contains a
// newline, the ScanLine will continue to return, requesting more data from the
// buffer, which can't increase in size anymore, causing a hang.
func dropCR(data []byte) []byte {
if len(data) > 0 && data[len(data)-1] == '\r' {
return data[0 : len(data)-1]
}
return data
}
func ScanLinesSmallerThanBuffer(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
// We have a full newline-terminated line.
return i + 1, dropCR(data[0:i]), nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), dropCR(data), nil
}
// Our tweak:
// Buffer is full, so we can't get more data. Just return what we have as
// its own token so we can keep going, even though there's no newline.
if len(data)+1 >= bufio.MaxScanTokenSize {
return len(data), data[0 : len(data)-1], nil
}
// Request more data.
return 0, nil, nil
}