Optimize JSON encoding even further

Last optimization was for JSON string with no character to encode. This
version focuses on strings with some chars to encode, trying to apply
the same trick for substrings that do not need encoding.

benchmark                old ns/op     new ns/op    delta
.../NoEncoding-8         60.2          51.3         -14.78%
.../EncodingFirst-8      140           116          -17.14%
.../EncodingMiddle-8     112           86.4         -22.86%
.../EncodingLast-8       62.8          61.1         -2.71%
.../MultiBytesFirst-8    164           129          -21.34%
.../MultiBytesMiddle-8   133           96.9         -27.14%
.../MultiBytesLast-8     81.9          73.5         -10.26%
This commit is contained in:
Olivier Poitrey 2017-06-24 20:55:52 -07:00
parent 9c5f03507d
commit 447d0fc7f5
3 changed files with 78 additions and 54 deletions

View File

@ -255,11 +255,11 @@ Some settings can be changed and will by applied to all loggers:
All operations are allocation free (those numbers *include* JSON encoding): All operations are allocation free (those numbers *include* JSON encoding):
``` ```
BenchmarkLogEmpty-8 100000000 19.1 ns/op 0 B/op 0 allocs/op BenchmarkLogEmpty-8 100000000 19.1 ns/op 0 B/op 0 allocs/op
BenchmarkDisabled-8 500000000 4.07 ns/op 0 B/op 0 allocs/op BenchmarkDisabled-8 500000000 4.07 ns/op 0 B/op 0 allocs/op
BenchmarkInfo-8 30000000 46.3 ns/op 0 B/op 0 allocs/op BenchmarkInfo-8 30000000 42.5 ns/op 0 B/op 0 allocs/op
BenchmarkContextFields-8 30000000 47.1 ns/op 0 B/op 0 allocs/op BenchmarkContextFields-8 30000000 44.9 ns/op 0 B/op 0 allocs/op
BenchmarkLogFields-8 10000000 186 ns/op 0 B/op 0 allocs/op BenchmarkLogFields-8 10000000 184 ns/op 0 B/op 0 allocs/op
``` ```
Using Uber's zap [comparison benchmark](https://github.com/uber-go/zap#performance): Using Uber's zap [comparison benchmark](https://github.com/uber-go/zap#performance):

106
json.go
View File

@ -21,48 +21,10 @@ func appendJSONString(dst []byte, s string) []byte {
// Check if the character needs encoding. Control characters, slashes, // Check if the character needs encoding. Control characters, slashes,
// and the double quote need json encoding. Bytes above the ascii // and the double quote need json encoding. Bytes above the ascii
// boundary needs utf8 encoding. // boundary needs utf8 encoding.
if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { if s[i] < 0x20 || s[i] > 0x7e || s[i] == '\\' || s[i] == '"' {
// We encountered a character that needs to be encoded. Let's // We encountered a character that needs to be encoded. Switch
// append the previous simple characters to the byte slice // to complex version of the algorithm.
// and switch our operation to read and encode the remainder dst = appendJSONStringComplex(dst, s, i)
// characters byte-by-byte.
dst = append(dst, s[:i]...)
for i < len(s) {
if b := s[i]; b < utf8.RuneSelf {
switch b {
case '"', '\\':
dst = append(dst, '\\', b)
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
default:
if b >= 0x20 {
dst = append(dst, b)
} else {
dst = append(dst, '\\', 'u', '0', '0',
hex[b>>4], hex[b&0xF])
}
}
i++
continue
}
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
dst = append(dst, `\ufffd`...)
i++
continue
}
dst = append(dst, s[i:i+size]...)
i += size
}
// End with a double quote
return append(dst, '"') return append(dst, '"')
} }
} }
@ -72,3 +34,63 @@ func appendJSONString(dst []byte, s string) []byte {
// End with a double quote // End with a double quote
return append(dst, '"') return append(dst, '"')
} }
// appendJSONStringComplex is used by appendJSONString to take over an in
// progress JSON string encoding that encountered a character that needs
// to be encoded.
func appendJSONStringComplex(dst []byte, s string, i int) []byte {
start := 0
for i < len(s) {
b := s[i]
if b >= utf8.RuneSelf {
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
// In case of error, first append previous simple characters to
// the byte slice if any and append a remplacement character code
// in place of the invalid sequence.
if start < i {
dst = append(dst, s[start:i]...)
}
dst = append(dst, `\ufffd`...)
i += size
start = i
continue
}
i += size
continue
}
if b >= 0x20 && b <= 0x7e && b != '\\' && b != '"' {
i++
continue
}
// We encountered a character that needs to be encoded.
// Let's append the previous simple characters to the byte slice
// and switch our operation to read and encode the remainder
// characters byte-by-byte.
if start < i {
dst = append(dst, s[start:i]...)
}
switch b {
case '"', '\\':
dst = append(dst, '\\', b)
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
default:
dst = append(dst, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF])
}
i++
start = i
}
if start < len(s) {
dst = append(dst, s[start:]...)
}
return dst
}

View File

@ -43,6 +43,8 @@ func TestAppendJSONString(t *testing.T) {
{"\x1d", `"\u001d"`}, {"\x1d", `"\u001d"`},
{"\x1e", `"\u001e"`}, {"\x1e", `"\u001e"`},
{"\x1f", `"\u001f"`}, {"\x1f", `"\u001f"`},
{"✭", `"✭"`},
{"foo\xc2\x7fbar", `"foo\ufffd\u007fbar"`}, // invalid sequence
{"ascii", `"ascii"`}, {"ascii", `"ascii"`},
{"\"a", `"\"a"`}, {"\"a", `"\"a"`},
{"\x1fa", `"\u001fa"`}, {"\x1fa", `"\u001fa"`},
@ -61,13 +63,13 @@ func TestAppendJSONString(t *testing.T) {
func BenchmarkAppendJSONString(b *testing.B) { func BenchmarkAppendJSONString(b *testing.B) {
tests := map[string]string{ tests := map[string]string{
"NoEncoding": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, "NoEncoding": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`,
"EncodingFirst": `"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, "EncodingFirst": `"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`,
"EncodingMiddle": `aaaaaaaaaaaaaaaaaaaaaaaaa"aaaaaaaaaaaaaaaaaaaaaaaa`, "EncodingMiddle": `aaaaaaaaaaaaaaaaaaaaaaaaa"aaaaaaaaaaaaaaaaaaaaaaaa`,
"EncodingLast": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, "EncodingLast": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
"RuneFirst": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, "MultiBytesFirst": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`,
"RuneMiddle": `aaaaaaaaaaaaaaaaaaaaaaaaa❤aaaaaaaaaaaaaaaaaaaaaaaa`, "MultiBytesMiddle": `aaaaaaaaaaaaaaaaaaaaaaaaa❤aaaaaaaaaaaaaaaaaaaaaaaa`,
"RuneLast": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa❤`, "MultiBytesLast": `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa❤`,
} }
for name, str := range tests { for name, str := range tests {
b.Run(name, func(b *testing.B) { b.Run(name, func(b *testing.B) {