a
/
zlog
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support for adding IP Address/Prefix + stream based decoder (#49) 

* added IPAddr, IPPrefix and stream based cbor decoder
* Update README with cbor decoder tool info
* Update README in cbor with comparison data
This commit is contained in:
Ravi Raju 2018-04-03 14:07:18 -07:00 committed by Olivier Poitrey
parent 05eafee0eb
commit 2ccfab3e07
22 changed files with 1191 additions and 685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -436,6 +436,9 @@ In addition to the default JSON encoding, `zerolog` can produce binary logs usin
go build -tags binary_log . go build -tags binary_log .
``` ```
To Decode binary encoded log files you can use any CBOR decoder. One has been tested to work
with zerolog library is CSD(https://github.com/toravir/csd/).
## Benchmarks ## Benchmarks
All operations are allocation free (those numbers *include* JSON encoding): All operations are allocation free (those numbers *include* JSON encoding):

19
array.go
View File

@ -1,6 +1,7 @@
package zerolog package zerolog
import ( import (
"net"
"sync" "sync"
"time" "time"
) )
@ -174,3 +175,21 @@ func (a *Array) Interface(i interface{}) *Array {
a.buf = appendInterface(appendArrayDelim(a.buf), i) a.buf = appendInterface(appendArrayDelim(a.buf), i)
return a return a
} }
// IPAddr adds IPv4 or IPv6 address to the array
func (a *Array) IPAddr(ip net.IP) *Array {
a.buf = appendIPAddr(appendArrayDelim(a.buf), ip)
return a
}
// IPPrefix adds IPv4 or IPv6 Prefix (IP + mask) to the array
func (a *Array) IPPrefix(pfx net.IPNet) *Array {
a.buf = appendIPPrefix(appendArrayDelim(a.buf), pfx)
return a
}
// MACAddr adds a MAC (Ethernet) address to the array
func (a *Array) MACAddr(ha net.HardwareAddr) *Array {
a.buf = appendMACAddr(appendArrayDelim(a.buf), ha)
return a
}

4
array_test.go
View File

@ -1,6 +1,7 @@
package zerolog package zerolog
import ( import (
"net"
"testing" "testing"
"time" "time"
) )
@ -24,8 +25,9 @@ func TestArray(t *testing.T) {
Bytes([]byte("b")). Bytes([]byte("b")).
Hex([]byte{0x1f}). Hex([]byte{0x1f}).
Time(time.Time{}). Time(time.Time{}).
IPAddr(net.IP{192, 168, 0, 10}).
Dur(0) Dur(0)
want := `[true,1,2,3,4,5,6,7,8,9,10,11,12,"a","b","1f","0001-01-01T00:00:00Z",0]` want := `[true,1,2,3,4,5,6,7,8,9,10,11,12,"a","b","1f","0001-01-01T00:00:00Z","192.168.0.10",0]`
if got := decodeObjectToStr(a.write([]byte{})); got != want { if got := decodeObjectToStr(a.write([]byte{})); got != want {
t.Errorf("Array.write()\ngot: %s\nwant: %s", got, want) t.Errorf("Array.write()\ngot: %s\nwant: %s", got, want)
} }

19
context.go
View File

@ -2,6 +2,7 @@ package zerolog
import ( import (
"io/ioutil" "io/ioutil"
"net"
"time" "time"
) )
@ -330,3 +331,21 @@ func (c Context) Caller() Context {
c.l = c.l.Hook(ch) c.l = c.l.Hook(ch)
return c return c
} }
// IPAddr adds IPv4 or IPv6 Address to the context
func (c Context) IPAddr(key string, ip net.IP) Context {
c.l.context = appendIPAddr(appendKey(c.l.context, key), ip)
return c
}
// IPPrefix adds IPv4 or IPv6 Prefix (address and mask) to the context
func (c Context) IPPrefix(key string, pfx net.IPNet) Context {
c.l.context = appendIPPrefix(appendKey(c.l.context, key), pfx)
return c
}
// MACAddr adds MAC address to the context
func (c Context) MACAddr(key string, ha net.HardwareAddr) Context {
c.l.context = appendMACAddr(appendKey(c.l.context, key), ha)
return c
}

13
encoder_cbor.go
View File

@ -5,6 +5,7 @@ package zerolog
// This file contains bindings to do binary encoding. // This file contains bindings to do binary encoding.
import ( import (
"net"
"time" "time"
"github.com/rs/zerolog/internal/cbor" "github.com/rs/zerolog/internal/cbor"
@ -211,6 +212,18 @@ func decodeObjectToStr(in []byte) string {
return cbor.DecodeObjectToStr(in) return cbor.DecodeObjectToStr(in)
} }
func appendIPAddr(dst []byte, ip net.IP) []byte {
return cbor.AppendIPAddr(dst, ip)
}
func appendIPPrefix(dst []byte, pfx net.IPNet) []byte {
return cbor.AppendIPPrefix(dst, pfx)
}
func appendMACAddr(dst []byte, ha net.HardwareAddr) []byte {
return cbor.AppendMACAddr(dst, ha)
}
// decodeIfBinaryToBytes - converts a binary formatted log msg to a // decodeIfBinaryToBytes - converts a binary formatted log msg to a
// JSON formatted Bytes Log message. // JSON formatted Bytes Log message.
func decodeIfBinaryToBytes(in []byte) []byte { func decodeIfBinaryToBytes(in []byte) []byte {

17
encoder_json.go
View File

@ -6,6 +6,7 @@ package zerolog
// JSON encoded byte stream. // JSON encoded byte stream.
import ( import (
"net"
"strconv" "strconv"
"time" "time"
@ -211,6 +212,18 @@ func decodeIfBinaryToBytes(in []byte) []byte {
return in return in
} }
func appendHex(in []byte, val []byte) []byte { func appendIPAddr(dst []byte, ip net.IP) []byte {
return json.AppendHex(in, val) return json.AppendIPAddr(dst, ip)
}
func appendIPPrefix(dst []byte, pfx net.IPNet) []byte {
return json.AppendIPPrefix(dst, pfx)
}
func appendMACAddr(dst []byte, ha net.HardwareAddr) []byte {
return json.AppendMACAddr(dst, ha)
}
func appendHex(in []byte, val []byte) []byte {
return json.AppendHex(in, val)
} }

28
event.go
View File

@ -2,6 +2,7 @@ package zerolog
import ( import (
"fmt" "fmt"
"net"
"os" "os"
"runtime" "runtime"
"strconv" "strconv"
@ -597,3 +598,30 @@ func (e *Event) caller(skip int) *Event {
e.buf = appendString(appendKey(e.buf, CallerFieldName), file+":"+strconv.Itoa(line)) e.buf = appendString(appendKey(e.buf, CallerFieldName), file+":"+strconv.Itoa(line))
return e return e
} }
// IPAddr adds IPv4 or IPv6 Address to the event
func (e *Event) IPAddr(key string, ip net.IP) *Event {
if e == nil {
return e
}
e.buf = appendIPAddr(appendKey(e.buf, key), ip)
return e
}
// IPPrefix adds IPv4 or IPv6 Prefix (address and mask) to the event
func (e *Event) IPPrefix(key string, pfx net.IPNet) *Event {
if e == nil {
return e
}
e.buf = appendIPPrefix(appendKey(e.buf, key), pfx)
return e
}
// MACAddr adds MAC address to the event
func (e *Event) MACAddr(key string, ha net.HardwareAddr) *Event {
if e == nil {
return e
}
e.buf = appendMACAddr(appendKey(e.buf, key), ha)
return e
}

7
fields.go
View File

@ -1,6 +1,7 @@
package zerolog package zerolog
import ( import (
"net"
"sort" "sort"
"time" "time"
) )
@ -118,6 +119,12 @@ func appendFields(dst []byte, fields map[string]interface{}) []byte {
dst = appendDurations(dst, val, DurationFieldUnit, DurationFieldInteger) dst = appendDurations(dst, val, DurationFieldUnit, DurationFieldInteger)
case nil: case nil:
dst = appendNil(dst) dst = appendNil(dst)
case net.IP:
dst = appendIPAddr(dst, val)
case net.IPNet:
dst = appendIPPrefix(dst, val)
case net.HardwareAddr:
dst = appendMACAddr(dst, val)
default: default:
dst = appendInterface(dst, val) dst = appendInterface(dst, val)
} }

122
internal/cbor/README.md
View File

@ -1,74 +1,56 @@
Reference: ## Reference:
CBOR Encoding is described in RFC7049 https://tools.ietf.org/html/rfc7049 CBOR Encoding is described in [RFC7049](https://tools.ietf.org/html/rfc7049)
## Comparison of JSON vs CBOR
Two main areas of reduction are:
1. CPU usage to write a log msg
2. Size (in bytes) of log messages.
Tests and benchmark: CPU Usage savings are below:
``` ```
sprint @ cbor>go test -v -benchmem -bench=. name JSON time/op CBOR time/op delta
=== RUN TestDecodeInteger Info-32 15.3ns ± 1% 11.7ns ± 3% -23.78% (p=0.000 n=9+10)
--- PASS: TestDecodeInteger (0.00s) ContextFields-32 16.2ns ± 2% 12.3ns ± 3% -23.97% (p=0.000 n=9+9)
=== RUN TestDecodeString ContextAppend-32 6.70ns ± 0% 6.20ns ± 0% -7.44% (p=0.000 n=9+9)
--- PASS: TestDecodeString (0.00s) LogFields-32 66.4ns ± 0% 24.6ns ± 2% -62.89% (p=0.000 n=10+9)
=== RUN TestDecodeArray LogArrayObject-32 911ns ±11% 768ns ± 6% -15.64% (p=0.000 n=10+10)
--- PASS: TestDecodeArray (0.00s) LogFieldType/Floats-32 70.3ns ± 2% 29.5ns ± 1% -57.98% (p=0.000 n=10+10)
=== RUN TestDecodeMap LogFieldType/Err-32 14.0ns ± 3% 12.1ns ± 8% -13.20% (p=0.000 n=8+10)
--- PASS: TestDecodeMap (0.00s) LogFieldType/Dur-32 17.2ns ± 2% 13.1ns ± 1% -24.27% (p=0.000 n=10+9)
=== RUN TestDecodeBool LogFieldType/Object-32 54.3ns ±11% 52.3ns ± 7% ~ (p=0.239 n=10+10)
--- PASS: TestDecodeBool (0.00s) LogFieldType/Ints-32 20.3ns ± 2% 15.1ns ± 2% -25.50% (p=0.000 n=9+10)
=== RUN TestDecodeFloat LogFieldType/Interfaces-32 642ns ±11% 621ns ± 9% ~ (p=0.118 n=10+10)
--- PASS: TestDecodeFloat (0.00s) LogFieldType/Interface(Objects)-32 635ns ±13% 632ns ± 9% ~ (p=0.592 n=10+10)
=== RUN TestDecodeTimestamp LogFieldType/Times-32 294ns ± 0% 27ns ± 1% -90.71% (p=0.000 n=10+9)
--- PASS: TestDecodeTimestamp (0.00s) LogFieldType/Durs-32 121ns ± 0% 33ns ± 2% -72.44% (p=0.000 n=9+9)
=== RUN TestDecodeCbor2Json LogFieldType/Interface(Object)-32 56.6ns ± 8% 52.3ns ± 8% -7.54% (p=0.007 n=10+10)
--- PASS: TestDecodeCbor2Json (0.00s) LogFieldType/Errs-32 17.8ns ± 3% 16.1ns ± 2% -9.71% (p=0.000 n=10+9)
=== RUN TestAppendString LogFieldType/Time-32 40.5ns ± 1% 12.7ns ± 6% -68.66% (p=0.000 n=8+9)
--- PASS: TestAppendString (0.00s) LogFieldType/Bool-32 12.0ns ± 5% 10.2ns ± 2% -15.18% (p=0.000 n=10+8)
=== RUN TestAppendBytes LogFieldType/Bools-32 17.2ns ± 2% 12.6ns ± 4% -26.63% (p=0.000 n=10+10)
--- PASS: TestAppendBytes (0.00s) LogFieldType/Int-32 12.3ns ± 2% 11.2ns ± 4% -9.27% (p=0.000 n=9+10)
=== RUN TestAppendTimeNow LogFieldType/Float-32 16.7ns ± 1% 12.6ns ± 2% -24.42% (p=0.000 n=7+9)
--- PASS: TestAppendTimeNow (0.00s) LogFieldType/Str-32 12.7ns ± 7% 11.3ns ± 7% -10.88% (p=0.000 n=10+9)
=== RUN TestAppendTimePastPresentInteger LogFieldType/Strs-32 20.3ns ± 3% 18.2ns ± 3% -10.25% (p=0.000 n=9+10)
--- PASS: TestAppendTimePastPresentInteger (0.00s) LogFieldType/Interface-32 183ns ±12% 175ns ± 9% ~ (p=0.078 n=10+10)
=== RUN TestAppendTimePastPresentFloat
--- PASS: TestAppendTimePastPresentFloat (0.00s)
=== RUN TestAppendNull
--- PASS: TestAppendNull (0.00s)
=== RUN TestAppendBool
--- PASS: TestAppendBool (0.00s)
=== RUN TestAppendBoolArray
--- PASS: TestAppendBoolArray (0.00s)
=== RUN TestAppendInt
--- PASS: TestAppendInt (0.00s)
=== RUN TestAppendIntArray
--- PASS: TestAppendIntArray (0.00s)
=== RUN TestAppendFloat32
--- PASS: TestAppendFloat32 (0.00s)
goos: linux
goarch: amd64
pkg: github.com/toravir/zerolog/internal/cbor
BenchmarkAppendString/MultiBytesLast-4 30000000 43.3 ns/op 0 B/op 0 allocs/op
BenchmarkAppendString/NoEncoding-4 30000000 48.2 ns/op 0 B/op 0 allocs/op
BenchmarkAppendString/EncodingFirst-4 30000000 48.2 ns/op 0 B/op 0 allocs/op
BenchmarkAppendString/EncodingMiddle-4 30000000 41.7 ns/op 0 B/op 0 allocs/op
BenchmarkAppendString/EncodingLast-4 30000000 51.8 ns/op 0 B/op 0 allocs/op
BenchmarkAppendString/MultiBytesFirst-4 50000000 38.0 ns/op 0 B/op 0 allocs/op
BenchmarkAppendString/MultiBytesMiddle-4 50000000 38.0 ns/op 0 B/op 0 allocs/op
BenchmarkAppendTime/Integer-4 50000000 39.6 ns/op 0 B/op 0 allocs/op
BenchmarkAppendTime/Float-4 30000000 56.1 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/uint8-4 50000000 29.1 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/uint16-4 50000000 30.3 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/uint32-4 50000000 37.1 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/int8-4 100000000 21.5 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/int16-4 50000000 25.8 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/int32-4 50000000 26.7 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/int-Positive-4 100000000 21.5 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/int-Negative-4 100000000 20.7 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/uint64-4 50000000 36.7 ns/op 0 B/op 0 allocs/op
BenchmarkAppendInt/int64-4 30000000 39.6 ns/op 0 B/op 0 allocs/op
BenchmarkAppendFloat/Float32-4 50000000 23.9 ns/op 0 B/op 0 allocs/op
BenchmarkAppendFloat/Float64-4 50000000 32.8 ns/op 0 B/op 0 allocs/op
PASS
ok github.com/toravir/zerolog/internal/cbor 34.969s
sprint @ cbor>
``` ```
Log message size savings is greatly dependent on the number and type of fields in the log message.
Assuming this log message (with an Integer, timestamp and string, in addition to level).
`{"level":"error","Fault":41650,"time":"2018-04-01T15:18:19-07:00","message":"Some Message"}`
Two measurements were done for the log file sizes - one without any compression, second
using [compress/zlib](https://golang.org/pkg/compress/zlib/).
Results for 10,000 log messages:
| Log Format | Plain File Size (in KB) | Compressed File Size (in KB) |
| :--- | :---: | :---: |
| JSON | 920 | 28 |
| CBOR | 550 | 28 |
The example used to calculate the above data is available in [Examples](examples).

25
internal/cbor/cbor.go
View File

@ -7,25 +7,34 @@ import "time"
const ( const (
majorOffset = 5 majorOffset = 5
additionalMax = 23 additionalMax = 23
//Non Values
// Non Values.
additionalTypeBoolFalse byte = 20 additionalTypeBoolFalse byte = 20
additionalTypeBoolTrue byte = 21 additionalTypeBoolTrue byte = 21
additionalTypeNull byte = 22 additionalTypeNull byte = 22
//Integer (+ve and -ve) Sub-types
// Integer (+ve and -ve) Sub-types.
additionalTypeIntUint8 byte = 24 additionalTypeIntUint8 byte = 24
additionalTypeIntUint16 byte = 25 additionalTypeIntUint16 byte = 25
additionalTypeIntUint32 byte = 26 additionalTypeIntUint32 byte = 26
additionalTypeIntUint64 byte = 27 additionalTypeIntUint64 byte = 27
//Float Sub-types
// Float Sub-types.
additionalTypeFloat16 byte = 25 additionalTypeFloat16 byte = 25
additionalTypeFloat32 byte = 26 additionalTypeFloat32 byte = 26
additionalTypeFloat64 byte = 27 additionalTypeFloat64 byte = 27
additionalTypeBreak byte = 31 additionalTypeBreak byte = 31
//Tag Sub-types
additionalTypeTimestamp byte = 01 // Tag Sub-types.
additionalTypeEmbeddedJSON byte = 31 additionalTypeTimestamp byte = 01
additionalTypeTagHexString uint16 = 262
//Unspecified number of elements // Extended Tags - from https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml
additionalTypeTagNetworkAddr uint16 = 260
additionalTypeTagNetworkPrefix uint16 = 261
additionalTypeEmbeddedJSON uint16 = 262
additionalTypeTagHexString uint16 = 263
// Unspecified number of elements.
additionalTypeInfiniteCount byte = 31 additionalTypeInfiniteCount byte = 31
) )
const ( const (

610
internal/cbor/decode_stream.go Normal file
View File

@ -0,0 +1,610 @@
package cbor
// This file contains code to decode a stream of CBOR Data into JSON.
import (
"bufio"
"bytes"
"fmt"
"io"
"math"
"net"
"runtime"
"strconv"
"strings"
"time"
"unicode/utf8"
)
var decodeTimeZone *time.Location
const hexTable = "0123456789abcdef"
func readNBytes(src *bufio.Reader, n int) []byte {
ret := make([]byte, n)
for i := 0; i < n; i++ {
ch, e := src.ReadByte()
if e != nil {
panic(fmt.Errorf("Tried to Read %d Bytes.. But hit end of file", n))
}
ret[i] = ch
}
return ret
}
func readByte(src *bufio.Reader) byte {
b, e := src.ReadByte()
if e != nil {
panic(fmt.Errorf("Tried to Read 1 Byte.. But hit end of file"))
}
return b
}
func decodeIntAdditonalType(src *bufio.Reader, minor byte) int64 {
val := int64(0)
if minor <= 23 {
val = int64(minor)
} else {
bytesToRead := 0
switch minor {
case additionalTypeIntUint8:
bytesToRead = 1
case additionalTypeIntUint16:
bytesToRead = 2
case additionalTypeIntUint32:
bytesToRead = 4
case additionalTypeIntUint64:
bytesToRead = 8
default:
panic(fmt.Errorf("Invalid Additional Type: %d in decodeInteger (expected <28)", minor))
}
pb := readNBytes(src, bytesToRead)
for i := 0; i < bytesToRead; i++ {
val = val * 256
val += int64(pb[i])
}
}
return val
}
func decodeInteger(src *bufio.Reader) int64 {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeUnsignedInt && major != majorTypeNegativeInt {
panic(fmt.Errorf("Major type is: %d in decodeInteger!! (expected 0 or 1)", major))
}
val := decodeIntAdditonalType(src, minor)
if major == 0 {
return val
}
return (-1 - val)
}
func decodeFloat(src *bufio.Reader) (float64, int) {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeSimpleAndFloat {
panic(fmt.Errorf("Incorrect Major type is: %d in decodeFloat", major))
}
switch minor {
case additionalTypeFloat16:
panic(fmt.Errorf("float16 is not suppported in decodeFloat"))
case additionalTypeFloat32:
pb := readNBytes(src, 4)
switch string(pb) {
case float32Nan:
return math.NaN(), 4
case float32PosInfinity:
return math.Inf(0), 4
case float32NegInfinity:
return math.Inf(-1), 4
}
n := uint32(0)
for i := 0; i < 4; i++ {
n = n * 256
n += uint32(pb[i])
}
val := math.Float32frombits(n)
return float64(val), 4
case additionalTypeFloat64:
pb := readNBytes(src, 8)
switch string(pb) {
case float64Nan:
return math.NaN(), 8
case float64PosInfinity:
return math.Inf(0), 8
case float64NegInfinity:
return math.Inf(-1), 8
}
n := uint64(0)
for i := 0; i < 8; i++ {
n = n * 256
n += uint64(pb[i])
}
val := math.Float64frombits(n)
return val, 8
}
panic(fmt.Errorf("Invalid Additional Type: %d in decodeFloat", minor))
}
func decodeStringComplex(dst []byte, s string, pos uint) []byte {
i := int(pos)
const hex = "0123456789abcdef"
start := 0
for i < len(s) {
b := s[i]
if b >= utf8.RuneSelf {
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
// In case of error, first append previous simple characters to
// the byte slice if any and append a replacement character code
// in place of the invalid sequence.
if start < i {
dst = append(dst, s[start:i]...)
}
dst = append(dst, `\ufffd`...)
i += size
start = i
continue
}
i += size
continue
}
if b >= 0x20 && b <= 0x7e && b != '\\' && b != '"' {
i++
continue
}
// We encountered a character that needs to be encoded.
// Let's append the previous simple characters to the byte slice
// and switch our operation to read and encode the remainder
// characters byte-by-byte.
if start < i {
dst = append(dst, s[start:i]...)
}
switch b {
case '"', '\\':
dst = append(dst, '\\', b)
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
default:
dst = append(dst, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF])
}
i++
start = i
}
if start < len(s) {
dst = append(dst, s[start:]...)
}
return dst
}
func decodeString(src *bufio.Reader, noQuotes bool) []byte {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeByteString {
panic(fmt.Errorf("Major type is: %d in decodeString", major))
}
result := []byte{}
if !noQuotes {
result = append(result, '"')
}
length := decodeIntAdditonalType(src, minor)
len := int(length)
pbs := readNBytes(src, len)
result = append(result, pbs...)
if noQuotes {
return result
}
return append(result, '"')
}
func decodeUTF8String(src *bufio.Reader) []byte {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeUtf8String {
panic(fmt.Errorf("Major type is: %d in decodeUTF8String", major))
}
result := []byte{'"'}
length := decodeIntAdditonalType(src, minor)
len := int(length)
pbs := readNBytes(src, len)
for i := 0; i < len; i++ {
// Check if the character needs encoding. Control characters, slashes,
// and the double quote need json encoding. Bytes above the ascii
// boundary needs utf8 encoding.
if pbs[i] < 0x20 || pbs[i] > 0x7e || pbs[i] == '\\' || pbs[i] == '"' {
// We encountered a character that needs to be encoded. Switch
// to complex version of the algorithm.
dst := []byte{'"'}
dst = decodeStringComplex(dst, string(pbs), uint(i))
return append(dst, '"')
}
}
// The string has no need for encoding an therefore is directly
// appended to the byte slice.
result = append(result, pbs...)
return append(result, '"')
}
func array2Json(src *bufio.Reader, dst io.Writer) {
dst.Write([]byte{'['})
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeArray {
panic(fmt.Errorf("Major type is: %d in array2Json", major))
}
len := 0
unSpecifiedCount := false
if minor == additionalTypeInfiniteCount {
unSpecifiedCount = true
} else {
length := decodeIntAdditonalType(src, minor)
len = int(length)
}
for i := 0; unSpecifiedCount || i < len; i++ {
if unSpecifiedCount {
pb, e := src.Peek(1)
if e != nil {
panic(e)
}
if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
readByte(src)
break
}
}
cbor2JsonOneObject(src, dst)
if unSpecifiedCount {
pb, e := src.Peek(1)
if e != nil {
panic(e)
}
if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
readByte(src)
break
}
dst.Write([]byte{','})
} else if i+1 < len {
dst.Write([]byte{','})
}
}
dst.Write([]byte{']'})
}
func map2Json(src *bufio.Reader, dst io.Writer) {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeMap {
panic(fmt.Errorf("Major type is: %d in map2Json", major))
}
len := 0
unSpecifiedCount := false
if minor == additionalTypeInfiniteCount {
unSpecifiedCount = true
} else {
length := decodeIntAdditonalType(src, minor)
len = int(length)
}
dst.Write([]byte{'{'})
for i := 0; unSpecifiedCount || i < len; i++ {
if unSpecifiedCount {
pb, e := src.Peek(1)
if e != nil {
panic(e)
}
if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
readByte(src)
break
}
}
cbor2JsonOneObject(src, dst)
if i%2 == 0 {
// Even position values are keys.
dst.Write([]byte{':'})
} else {
if unSpecifiedCount {
pb, e := src.Peek(1)
if e != nil {
panic(e)
}
if pb[0] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
readByte(src)
break
}
dst.Write([]byte{','})
} else if i+1 < len {
dst.Write([]byte{','})
}
}
}
dst.Write([]byte{'}'})
}
func decodeTagData(src *bufio.Reader) []byte {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeTags {
panic(fmt.Errorf("Major type is: %d in decodeTagData", major))
}
switch minor {
case additionalTypeTimestamp:
return decodeTimeStamp(src)
// Tag value is larger than 256 (so uint16).
case additionalTypeIntUint16:
val := decodeIntAdditonalType(src, minor)
switch uint16(val) {
case additionalTypeEmbeddedJSON:
pb := readByte(src)
dataMajor := pb & maskOutAdditionalType
if dataMajor != majorTypeByteString {
panic(fmt.Errorf("Unsupported embedded Type: %d in decodeEmbeddedJSON", dataMajor))
}
src.UnreadByte()
return decodeString(src, true)
case additionalTypeTagNetworkAddr:
octets := decodeString(src, true)
ss := []byte{'"'}
switch len(octets) {
case 6: // MAC address.
ha := net.HardwareAddr(octets)
ss = append(append(ss, ha.String()...), '"')
case 4: // IPv4 address.
fallthrough
case 16: // IPv6 address.
ip := net.IP(octets)
ss = append(append(ss, ip.String()...), '"')
default:
panic(fmt.Errorf("Unexpected Network Address length: %d (expected 4,6,16)", len(octets)))
}
return ss
case additionalTypeTagNetworkPrefix:
pb := readByte(src)
if pb != byte(majorTypeMap|0x1) {
panic(fmt.Errorf("IP Prefix is NOT of MAP of 1 elements as expected"))
}
octets := decodeString(src, true)
val := decodeInteger(src)
ip := net.IP(octets)
var mask net.IPMask
pfxLen := int(val)
if len(octets) == 4 {
mask = net.CIDRMask(pfxLen, 32)
} else {
mask = net.CIDRMask(pfxLen, 128)
}
ipPfx := net.IPNet{IP: ip, Mask: mask}
ss := []byte{'"'}
ss = append(append(ss, ipPfx.String()...), '"')
return ss
case additionalTypeTagHexString:
octets := decodeString(src, true)
ss := []byte{'"'}
for _, v := range octets {
ss = append(ss, hexTable[v>>4], hexTable[v&0x0f])
}
return append(ss, '"')
default:
panic(fmt.Errorf("Unsupported Additional Tag Type: %d in decodeTagData", val))
}
}
panic(fmt.Errorf("Unsupported Additional Type: %d in decodeTagData", minor))
}
func decodeTimeStamp(src *bufio.Reader) []byte {
pb := readByte(src)
src.UnreadByte()
tsMajor := pb & maskOutAdditionalType
if tsMajor == majorTypeUnsignedInt || tsMajor == majorTypeNegativeInt {
n := decodeInteger(src)
t := time.Unix(n, 0)
if decodeTimeZone != nil {
t = t.In(decodeTimeZone)
} else {
t = t.In(time.UTC)
}
tsb := []byte{}
tsb = append(tsb, '"')
tsb = t.AppendFormat(tsb, IntegerTimeFieldFormat)
tsb = append(tsb, '"')
return tsb
} else if tsMajor == majorTypeSimpleAndFloat {
n, _ := decodeFloat(src)
secs := int64(n)
n -= float64(secs)
n *= float64(1e9)
t := time.Unix(secs, int64(n))
if decodeTimeZone != nil {
t = t.In(decodeTimeZone)
} else {
t = t.In(time.UTC)
}
tsb := []byte{}
tsb = append(tsb, '"')
tsb = t.AppendFormat(tsb, NanoTimeFieldFormat)
tsb = append(tsb, '"')
return tsb
}
panic(fmt.Errorf("TS format is neigther int nor float: %d", tsMajor))
}
func decodeSimpleFloat(src *bufio.Reader) []byte {
pb := readByte(src)
major := pb & maskOutAdditionalType
minor := pb & maskOutMajorType
if major != majorTypeSimpleAndFloat {
panic(fmt.Errorf("Major type is: %d in decodeSimpleFloat", major))
}
switch minor {
case additionalTypeBoolTrue:
return []byte("true")
case additionalTypeBoolFalse:
return []byte("false")
case additionalTypeNull:
return []byte("null")
case additionalTypeFloat16:
fallthrough
case additionalTypeFloat32:
fallthrough
case additionalTypeFloat64:
src.UnreadByte()
v, bc := decodeFloat(src)
ba := []byte{}
switch {
case math.IsNaN(v):
return []byte("\"NaN\"")
case math.IsInf(v, 1):
return []byte("\"+Inf\"")
case math.IsInf(v, -1):
return []byte("\"-Inf\"")
}
if bc == 5 {
ba = strconv.AppendFloat(ba, v, 'f', -1, 32)
} else {
ba = strconv.AppendFloat(ba, v, 'f', -1, 64)
}
return ba
default:
panic(fmt.Errorf("Invalid Additional Type: %d in decodeSimpleFloat", minor))
}
}
func cbor2JsonOneObject(src *bufio.Reader, dst io.Writer) {
pb, e := src.Peek(1)
if e != nil {
panic(e)
}
major := (pb[0] & maskOutAdditionalType)
switch major {
case majorTypeUnsignedInt:
fallthrough
case majorTypeNegativeInt:
n := decodeInteger(src)
dst.Write([]byte(strconv.Itoa(int(n))))
case majorTypeByteString:
s := decodeString(src, false)
dst.Write(s)
case majorTypeUtf8String:
s := decodeUTF8String(src)
dst.Write(s)
case majorTypeArray:
array2Json(src, dst)
case majorTypeMap:
map2Json(src, dst)
case majorTypeTags:
s := decodeTagData(src)
dst.Write(s)
case majorTypeSimpleAndFloat:
s := decodeSimpleFloat(src)
dst.Write(s)
}
}
func moreBytesToRead(src *bufio.Reader) bool {
_, e := src.ReadByte()
if e == nil {
src.UnreadByte()
return true
}
return false
}
// Cbor2JsonManyObjects decodes all the CBOR Objects read from src
// reader. It keeps on decoding until reader returns EOF (error when reading).
// Decoded string is written to the dst. At the end of every CBOR Object
// newline is written to the output stream.
//
// Returns error (if any) that was encountered during decode.
// The child functions will generate a panic when error is encountered and
// this function will recover non-runtime Errors and return the reason as error.
func Cbor2JsonManyObjects(src io.Reader, dst io.Writer) (err error) {
defer func() {
if r := recover(); r != nil {
if _, ok := r.(runtime.Error); ok {
panic(r)
}
err = r.(error)
}
}()
bufRdr := bufio.NewReader(src)
for moreBytesToRead(bufRdr) {
cbor2JsonOneObject(bufRdr, dst)
dst.Write([]byte("\n"))
}
return nil
}
// Detect if the bytes to be printed is Binary or not.
func binaryFmt(p []byte) bool {
if len(p) > 0 && p[0] > 0x7F {
return true
}
return false
}
func getReader(str string) *bufio.Reader {
return bufio.NewReader(strings.NewReader(str))
}
// DecodeIfBinaryToString converts a binary formatted log msg to a
// JSON formatted String Log message - suitable for printing to Console/Syslog.
func DecodeIfBinaryToString(in []byte) string {
if binaryFmt(in) {
var b bytes.Buffer
Cbor2JsonManyObjects(strings.NewReader(string(in)), &b)
return b.String()
}
return string(in)
}
// DecodeObjectToStr checks if the input is a binary format, if so,
// it will decode a single Object and return the decoded string.
func DecodeObjectToStr(in []byte) string {
if binaryFmt(in) {
var b bytes.Buffer
cbor2JsonOneObject(getReader(string(in)), &b)
return b.String()
}
return string(in)
}
// DecodeIfBinaryToBytes checks if the input is a binary format, if so,
// it will decode all Objects and return the decoded string as byte array.
func DecodeIfBinaryToBytes(in []byte) []byte {
if binaryFmt(in) {
var b bytes.Buffer
Cbor2JsonManyObjects(bytes.NewReader(in), &b)
return b.Bytes()
}
return in
}

548
internal/cbor/decoder.go
View File

@ -1,548 +0,0 @@
package cbor
import (
"bytes"
"fmt"
"io"
"math"
"strconv"
"time"
"unicode/utf8"
)
var decodeTimeZone *time.Location
const hexTable = "0123456789abcdef"
func decodeIntAdditonalType(src []byte, minor byte) (int64, uint, error) {
val := int64(0)
bytesRead := 0
if minor <= 23 {
val = int64(minor)
bytesRead = 0
} else {
switch minor {
case additionalTypeIntUint8:
bytesRead = 1
case additionalTypeIntUint16:
bytesRead = 2
case additionalTypeIntUint32:
bytesRead = 4
case additionalTypeIntUint64:
bytesRead = 8
default:
return 0, 0, fmt.Errorf("Invalid Additional Type: %d in decodeInteger (expected <28)", minor)
}
for i := 0; i < bytesRead; i++ {
val = val * 256
val += int64(src[i])
}
}
return val, uint(bytesRead), nil
}
func decodeInteger(src []byte) (int64, uint, error) {
major := src[0] & maskOutAdditionalType
minor := src[0] & maskOutMajorType
if major != majorTypeUnsignedInt && major != majorTypeNegativeInt {
return 0, 0, fmt.Errorf("Major type is: %d in decodeInteger!! (expected 0 or 1)", major)
}
val, bytesRead, err := decodeIntAdditonalType(src[1:], minor)
if err != nil {
return 0, 0, err
}
if major == 0 {
return val, 1 + bytesRead, nil
}
return (-1 - val), 1 + bytesRead, nil
}
func decodeFloat(src []byte) (float64, uint, error) {
major := (src[0] & maskOutAdditionalType)
minor := src[0] & maskOutMajorType
if major != majorTypeSimpleAndFloat {
return 0, 0, fmt.Errorf("Incorrect Major type is: %d in decodeFloat", major)
}
switch minor {
case additionalTypeFloat16:
return 0, 0, fmt.Errorf("float16 is not suppported in decodeFloat")
case additionalTypeFloat32:
switch string(src[1:5]) {
case float32Nan:
return math.NaN(), 5, nil
case float32PosInfinity:
return math.Inf(0), 5, nil
case float32NegInfinity:
return math.Inf(-1), 5, nil
}
n := uint32(0)
for i := 0; i < 4; i++ {
n = n * 256
n += uint32(src[i+1])
}
val := math.Float32frombits(n)
return float64(val), 5, nil
case additionalTypeFloat64:
switch string(src[1:9]) {
case float64Nan:
return math.NaN(), 9, nil
case float64PosInfinity:
return math.Inf(0), 9, nil
case float64NegInfinity:
return math.Inf(-1), 9, nil
}
n := uint64(0)
for i := 0; i < 8; i++ {
n = n * 256
n += uint64(src[i+1])
}
val := math.Float64frombits(n)
return val, 9, nil
}
return 0, 0, fmt.Errorf("Invalid Additional Type: %d in decodeFloat", minor)
}
func decodeStringComplex(dst []byte, s string, pos uint) []byte {
i := int(pos)
const hex = "0123456789abcdef"
start := 0
for i < len(s) {
b := s[i]
if b >= utf8.RuneSelf {
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
// In case of error, first append previous simple characters to
// the byte slice if any and append a replacement character code
// in place of the invalid sequence.
if start < i {
dst = append(dst, s[start:i]...)
}
dst = append(dst, `\ufffd`...)
i += size
start = i
continue
}
i += size
continue
}
if b >= 0x20 && b <= 0x7e && b != '\\' && b != '"' {
i++
continue
}
// We encountered a character that needs to be encoded.
// Let's append the previous simple characters to the byte slice
// and switch our operation to read and encode the remainder
// characters byte-by-byte.
if start < i {
dst = append(dst, s[start:i]...)
}
switch b {
case '"', '\\':
dst = append(dst, '\\', b)
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
default:
dst = append(dst, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF])
}
i++
start = i
}
if start < len(s) {
dst = append(dst, s[start:]...)
}
return dst
}
func decodeString(src []byte, noQuotes bool) ([]byte, uint, error) {
major := src[0] & maskOutAdditionalType
minor := src[0] & maskOutMajorType
if major != majorTypeByteString {
return []byte{}, 0, fmt.Errorf("Major type is: %d in decodeString", major)
}
result := []byte{'"'}
if noQuotes {
result = []byte{}
}
length, bytesRead, err := decodeIntAdditonalType(src[1:], minor)
if err != nil {
return []byte{}, 0, err
}
bytesRead++
st := bytesRead
len := uint(length)
bytesRead += len
result = append(result, src[st:st+len]...)
if noQuotes {
return result, bytesRead, nil
}
return append(result, '"'), bytesRead, nil
}
func decodeUTF8String(src []byte) ([]byte, uint, error) {
major := src[0] & maskOutAdditionalType
minor := src[0] & maskOutMajorType
if major != majorTypeUtf8String {
return []byte{}, 0, fmt.Errorf("Major type is: %d in decodeUTF8String", major)
}
result := []byte{'"'}
length, bytesRead, err := decodeIntAdditonalType(src[1:], minor)
if err != nil {
return []byte{}, 0, err
}
bytesRead++
st := bytesRead
len := uint(length)
bytesRead += len
for i := st; i < bytesRead; i++ {
// Check if the character needs encoding. Control characters, slashes,
// and the double quote need json encoding. Bytes above the ascii
// boundary needs utf8 encoding.
if src[i] < 0x20 || src[i] > 0x7e || src[i] == '\\' || src[i] == '"' {
// We encountered a character that needs to be encoded. Switch
// to complex version of the algorithm.
dst := []byte{'"'}
dst = decodeStringComplex(dst, string(src[st:st+len]), i-st)
return append(dst, '"'), bytesRead, nil
}
}
// The string has no need for encoding an therefore is directly
// appended to the byte slice.
result = append(result, src[st:st+len]...)
return append(result, '"'), bytesRead, nil
}
func array2Json(src []byte, dst io.Writer) (uint, error) {
dst.Write([]byte{'['})
major := (src[0] & maskOutAdditionalType)
minor := src[0] & maskOutMajorType
if major != majorTypeArray {
return 0, fmt.Errorf("Major type is: %d in array2Json", major)
}
len := 0
bytesRead := uint(0)
unSpecifiedCount := false
if minor == additionalTypeInfiniteCount {
unSpecifiedCount = true
bytesRead = 1
} else {
var length int64
var err error
length, bytesRead, err = decodeIntAdditonalType(src[1:], minor)
if err != nil {
fmt.Println("Error!!!")
return 0, err
}
len = int(length)
bytesRead++
}
curPos := bytesRead
for i := 0; unSpecifiedCount || i < len; i++ {
bc, err := Cbor2JsonOneObject(src[curPos:], dst)
if err != nil {
if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
bytesRead++
break
}
return 0, err
}
curPos += bc
bytesRead += bc
if unSpecifiedCount {
if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
bytesRead++
break
}
dst.Write([]byte{','})
} else if i+1 < len {
dst.Write([]byte{','})
}
}
dst.Write([]byte{']'})
return bytesRead, nil
}
func map2Json(src []byte, dst io.Writer) (uint, error) {
major := (src[0] & maskOutAdditionalType)
minor := src[0] & maskOutMajorType
if major != majorTypeMap {
return 0, fmt.Errorf("Major type is: %d in map2Json", major)
}
len := 0
bytesRead := uint(0)
unSpecifiedCount := false
if minor == additionalTypeInfiniteCount {
unSpecifiedCount = true
bytesRead = 1
} else {
var length int64
var err error
length, bytesRead, err = decodeIntAdditonalType(src[1:], minor)
if err != nil {
fmt.Println("Error!!!")
return 0, err
}
len = int(length)
bytesRead++
}
if len%2 == 1 {
return 0, fmt.Errorf("Invalid Length of map %d - has to be even", len)
}
dst.Write([]byte{'{'})
curPos := bytesRead
for i := 0; unSpecifiedCount || i < len; i++ {
bc, err := Cbor2JsonOneObject(src[curPos:], dst)
if err != nil {
//We hit the BREAK
if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
bytesRead++
break
}
return 0, err
}
curPos += bc
bytesRead += bc
if i%2 == 0 {
//Even position values are keys
dst.Write([]byte{':'})
} else {
if unSpecifiedCount {
if src[curPos] == byte(majorTypeSimpleAndFloat|additionalTypeBreak) {
bytesRead++
break
}
dst.Write([]byte{','})
} else if i+1 < len {
dst.Write([]byte{','})
}
}
}
dst.Write([]byte{'}'})
return bytesRead, nil
}
func decodeTagData(src []byte) ([]byte, uint, error) {
major := (src[0] & maskOutAdditionalType)
minor := src[0] & maskOutMajorType
if major != majorTypeTags {
return nil, 0, fmt.Errorf("Major type is: %d in decodeTagData", major)
}
if minor == additionalTypeTimestamp {
tsMajor := src[1] & maskOutAdditionalType
if tsMajor == majorTypeUnsignedInt || tsMajor == majorTypeNegativeInt {
n, bc, err := decodeInteger(src[1:])
if err != nil {
return []byte{}, 0, err
}
t := time.Unix(n, 0)
if decodeTimeZone != nil {
t = t.In(decodeTimeZone)
} else {
t = t.In(time.UTC)
}
tsb := []byte{}
tsb = append(tsb, '"')
tsb = t.AppendFormat(tsb, IntegerTimeFieldFormat)
tsb = append(tsb, '"')
return tsb, 1 + bc, nil
} else if tsMajor == majorTypeSimpleAndFloat {
n, bc, err := decodeFloat(src[1:])
if err != nil {
return []byte{}, 0, err
}
secs := int64(n)
n -= float64(secs)
n *= float64(1e9)
t := time.Unix(secs, int64(n))
if decodeTimeZone != nil {
t = t.In(decodeTimeZone)
} else {
t = t.In(time.UTC)
}
tsb := []byte{}
tsb = append(tsb, '"')
tsb = t.AppendFormat(tsb, NanoTimeFieldFormat)
tsb = append(tsb, '"')
return tsb, 1 + bc, nil
} else {
return nil, 0, fmt.Errorf("TS format is neigther int nor float: %d", tsMajor)
}
} else if minor == additionalTypeEmbeddedJSON {
dataMajor := src[1] & maskOutAdditionalType
if dataMajor == majorTypeByteString {
emb, bc, err := decodeString(src[1:], true)
if err != nil {
return nil, 0, err
}
return emb, 1 + bc, nil
}
return nil, 0, fmt.Errorf("Unsupported embedded Type: %d in decodeEmbeddedJSON", dataMajor)
} else if minor == additionalTypeIntUint16 {
val,_,_ := decodeIntAdditonalType(src[1:], minor)
if uint16(val) == additionalTypeTagHexString {
emb, bc, _ := decodeString(src[3:], true)
dst := []byte{'"'}
for _, v := range emb {
dst = append(dst, hexTable[v>>4], hexTable[v&0x0f])
}
return append(dst, '"'), 3+bc, nil
}
}
return nil, 0, fmt.Errorf("Unsupported Additional Type: %d in decodeTagData", minor)
}
func decodeSimpleFloat(src []byte) ([]byte, uint, error) {
major := (src[0] & maskOutAdditionalType)
minor := src[0] & maskOutMajorType
if major != majorTypeSimpleAndFloat {
return nil, 0, fmt.Errorf("Major type is: %d in decodeSimpleFloat", major)
}
switch minor {
case additionalTypeBoolTrue:
return []byte("true"), 1, nil
case additionalTypeBoolFalse:
return []byte("false"), 1, nil
case additionalTypeNull:
return []byte("null"), 1, nil
case additionalTypeFloat16:
fallthrough
case additionalTypeFloat32:
fallthrough
case additionalTypeFloat64:
v, bc, err := decodeFloat(src)
if err != nil {
return nil, 0, err
}
ba := []byte{}
switch {
case math.IsNaN(v):
return []byte("\"NaN\""), bc, nil
case math.IsInf(v, 1):
return []byte("\"+Inf\""), bc, nil
case math.IsInf(v, -1):
return []byte("\"-Inf\""), bc, nil
}
if bc == 5 {
ba = strconv.AppendFloat(ba, v, 'f', -1, 32)
} else {
ba = strconv.AppendFloat(ba, v, 'f', -1, 64)
}
return ba, bc, nil
default:
return nil, 0, fmt.Errorf("Invalid Additional Type: %d in decodeSimpleFloat", minor)
}
}
// Cbor2JsonOneObject takes in byte array and decodes ONE CBOR Object
// usually a MAP. Use this when only ONE CBOR object needs decoding.
// Decoded string is written to the dst.
// Returns the bytes decoded and if any error was encountered.
func Cbor2JsonOneObject(src []byte, dst io.Writer) (uint, error) {
var err error
major := (src[0] & maskOutAdditionalType)
bc := uint(0)
var s []byte
switch major {
case majorTypeUnsignedInt:
fallthrough
case majorTypeNegativeInt:
var n int64
n, bc, err = decodeInteger(src)
dst.Write([]byte(strconv.Itoa(int(n))))
case majorTypeByteString:
s, bc, err = decodeString(src, false)
dst.Write(s)
case majorTypeUtf8String:
s, bc, err = decodeUTF8String(src)
dst.Write(s)
case majorTypeArray:
bc, err = array2Json(src, dst)
case majorTypeMap:
bc, err = map2Json(src, dst)
case majorTypeTags:
s, bc, err = decodeTagData(src)
dst.Write(s)
case majorTypeSimpleAndFloat:
s, bc, err = decodeSimpleFloat(src)
dst.Write(s)
}
return bc, err
}
// Cbor2JsonManyObjects decodes all the CBOR Objects present in the
// source byte array. It keeps on decoding until it runs out of bytes.
// Decoded string is written to the dst. At the end of every CBOR Object
// newline is written to the output stream.
// Returns the number of bytes decoded and if any error was encountered.
func Cbor2JsonManyObjects(src []byte, dst io.Writer) (uint, error) {
curPos := uint(0)
totalBytes := uint(len(src))
for curPos < totalBytes {
bc, err := Cbor2JsonOneObject(src[curPos:], dst)
if err != nil {
return curPos, err
}
dst.Write([]byte("\n"))
curPos += bc
}
return curPos, nil
}
// Detect if the bytes to be printed is Binary or not.
func binaryFmt(p []byte) bool {
if len(p) > 0 && p[0] > 0x7F {
return true
}
return false
}
// DecodeIfBinaryToString converts a binary formatted log msg to a
// JSON formatted String Log message - suitable for printing to Console/Syslog.
func DecodeIfBinaryToString(in []byte) string {
if binaryFmt(in) {
var b bytes.Buffer
Cbor2JsonManyObjects(in, &b)
return b.String()
}
return string(in)
}
// DecodeObjectToStr checks if the input is a binary format, if so,
// it will decode a single Object and return the decoded string.
func DecodeObjectToStr(in []byte) string {
if binaryFmt(in) {
var b bytes.Buffer
Cbor2JsonOneObject(in, &b)
return b.String()
}
return string(in)
}
// DecodeIfBinaryToBytes checks if the input is a binary format, if so,
// it will decode all Objects and return the decoded string as byte array.
func DecodeIfBinaryToBytes(in []byte) []byte {
if binaryFmt(in) {
var b bytes.Buffer
Cbor2JsonManyObjects(in, &b)
return b.Bytes()
}
return in
}

113
internal/cbor/decoder_test.go
View File

@ -9,8 +9,8 @@ import (
func TestDecodeInteger(t *testing.T) { func TestDecodeInteger(t *testing.T) {
for _, tc := range integerTestCases { for _, tc := range integerTestCases {
gotv, gotc, err := decodeInteger([]byte(tc.binary)) gotv := decodeInteger(getReader(tc.binary))
if gotv != int64(tc.val) || int(gotc) != len(tc.binary) || err != nil { if gotv != int64(tc.val) {
t.Errorf("decodeInteger(0x%s)=0x%d, want: 0x%d", t.Errorf("decodeInteger(0x%s)=0x%d, want: 0x%d",
hex.EncodeToString([]byte(tc.binary)), gotv, tc.val) hex.EncodeToString([]byte(tc.binary)), gotv, tc.val)
} }
@ -19,10 +19,7 @@ func TestDecodeInteger(t *testing.T) {
func TestDecodeString(t *testing.T) { func TestDecodeString(t *testing.T) {
for _, tt := range encodeStringTests { for _, tt := range encodeStringTests {
got, _, err := decodeUTF8String([]byte(tt.binary)) got := decodeUTF8String(getReader(tt.binary))
if err != nil {
t.Errorf("Got Error for the case: %s", hex.EncodeToString([]byte(tt.binary)))
}
if string(got) != "\""+tt.json+"\"" { if string(got) != "\""+tt.json+"\"" {
t.Errorf("DecodeString(0x%s)=%s, want:\"%s\"\n", hex.EncodeToString([]byte(tt.binary)), string(got), t.Errorf("DecodeString(0x%s)=%s, want:\"%s\"\n", hex.EncodeToString([]byte(tt.binary)), string(got),
hex.EncodeToString([]byte(tt.json))) hex.EncodeToString([]byte(tt.json)))
@ -33,10 +30,7 @@ func TestDecodeString(t *testing.T) {
func TestDecodeArray(t *testing.T) { func TestDecodeArray(t *testing.T) {
for _, tc := range integerArrayTestCases { for _, tc := range integerArrayTestCases {
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
_, err := array2Json([]byte(tc.binary), buf) array2Json(getReader(tc.binary), buf)
if err != nil {
panic(err)
}
if buf.String() != tc.json { if buf.String() != tc.json {
t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.binary)), buf.String(), tc.json) t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.binary)), buf.String(), tc.json)
} }
@ -54,20 +48,14 @@ func TestDecodeArray(t *testing.T) {
} }
for _, tc := range infiniteArrayTestCases { for _, tc := range infiniteArrayTestCases {
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
_, err := array2Json([]byte(tc.in), buf) array2Json(getReader(tc.in), buf)
if err != nil {
panic(err)
}
if buf.String() != tc.out { if buf.String() != tc.out {
t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.out)), buf.String(), tc.out) t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.out)), buf.String(), tc.out)
} }
} }
for _, tc := range booleanArrayTestCases { for _, tc := range booleanArrayTestCases {
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
_, err := array2Json([]byte(tc.binary), buf) array2Json(getReader(tc.binary), buf)
if err != nil {
t.Errorf("array2Json(0x%s) errored out: %s", hex.EncodeToString([]byte(tc.binary)), err.Error())
}
if buf.String() != tc.json { if buf.String() != tc.json {
t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.binary)), buf.String(), tc.json) t.Errorf("array2Json(0x%s)=%s, want: %s", hex.EncodeToString([]byte(tc.binary)), buf.String(), tc.json)
} }
@ -94,20 +82,14 @@ var mapDecodeTestCases = []struct {
func TestDecodeMap(t *testing.T) { func TestDecodeMap(t *testing.T) {
for _, tc := range mapDecodeTestCases { for _, tc := range mapDecodeTestCases {
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
_, err := map2Json(tc.bin, buf) map2Json(getReader(string(tc.bin)), buf)
if err != nil {
t.Errorf("map2Json(0x%s) returned error", err)
}
if buf.String() != tc.json { if buf.String() != tc.json {
t.Errorf("map2Json(0x%s)=%s, want: %s", hex.EncodeToString(tc.bin), buf.String(), tc.json) t.Errorf("map2Json(0x%s)=%s, want: %s", hex.EncodeToString(tc.bin), buf.String(), tc.json)
} }
} }
for _, tc := range infiniteMapDecodeTestCases { for _, tc := range infiniteMapDecodeTestCases {
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
_, err := map2Json(tc.bin, buf) map2Json(getReader(string(tc.bin)), buf)
if err != nil {
t.Errorf("map2Json(0x%s) returned error", err)
}
if buf.String() != tc.json { if buf.String() != tc.json {
t.Errorf("map2Json(0x%s)=%s, want: %s", hex.EncodeToString(tc.bin), buf.String(), tc.json) t.Errorf("map2Json(0x%s)=%s, want: %s", hex.EncodeToString(tc.bin), buf.String(), tc.json)
} }
@ -116,10 +98,7 @@ func TestDecodeMap(t *testing.T) {
func TestDecodeBool(t *testing.T) { func TestDecodeBool(t *testing.T) {
for _, tc := range booleanTestCases { for _, tc := range booleanTestCases {
got, _, err := decodeSimpleFloat([]byte(tc.binary)) got := decodeSimpleFloat(getReader(tc.binary))
if err != nil {
t.Errorf("decodeSimpleFloat(0x%s) errored %s", hex.EncodeToString([]byte(tc.binary)), err.Error())
}
if string(got) != tc.json { if string(got) != tc.json {
t.Errorf("decodeSimpleFloat(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), string(got), tc.json) t.Errorf("decodeSimpleFloat(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), string(got), tc.json)
} }
@ -128,10 +107,7 @@ func TestDecodeBool(t *testing.T) {
func TestDecodeFloat(t *testing.T) { func TestDecodeFloat(t *testing.T) {
for _, tc := range float32TestCases { for _, tc := range float32TestCases {
got, _, err := decodeFloat([]byte(tc.binary)) got, _ := decodeFloat(getReader(tc.binary))
if err != nil {
t.Errorf("decodeFloat(0x%s) returned error: %s", hex.EncodeToString([]byte(tc.binary)), err.Error())
}
if got != float64(tc.val) { if got != float64(tc.val) {
t.Errorf("decodeFloat(0x%s)=%f, want:%f", hex.EncodeToString([]byte(tc.binary)), got, tc.val) t.Errorf("decodeFloat(0x%s)=%f, want:%f", hex.EncodeToString([]byte(tc.binary)), got, tc.val)
} }
@ -141,19 +117,13 @@ func TestDecodeFloat(t *testing.T) {
func TestDecodeTimestamp(t *testing.T) { func TestDecodeTimestamp(t *testing.T) {
decodeTimeZone, _ = time.LoadLocation("UTC") decodeTimeZone, _ = time.LoadLocation("UTC")
for _, tc := range timeIntegerTestcases { for _, tc := range timeIntegerTestcases {
tm, _, err := decodeTagData([]byte(tc.binary)) tm := decodeTagData(getReader(tc.binary))
if err != nil {
t.Errorf("decodeTagData(0x%s) returned error: %s", hex.EncodeToString([]byte(tc.binary)), err.Error())
}
if string(tm) != "\""+tc.rfcStr+"\"" { if string(tm) != "\""+tc.rfcStr+"\"" {
t.Errorf("decodeFloat(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), tm, tc.rfcStr) t.Errorf("decodeFloat(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), tm, tc.rfcStr)
} }
} }
for _, tc := range timeFloatTestcases { for _, tc := range timeFloatTestcases {
tm, _, err := decodeTagData([]byte(tc.out)) tm := decodeTagData(getReader(tc.out))
if err != nil {
t.Errorf("decodeTagData(0x%s) returned error: %s", hex.EncodeToString([]byte(tc.out)), err.Error())
}
//Since we convert to float and back - it may be slightly off - so //Since we convert to float and back - it may be slightly off - so
//we cannot check for exact equality instead, we'll check it is //we cannot check for exact equality instead, we'll check it is
//very close to each other Less than a Microsecond (lets not yet do nanosec) //very close to each other Less than a Microsecond (lets not yet do nanosec)
@ -166,6 +136,33 @@ func TestDecodeTimestamp(t *testing.T) {
} }
} }
func TestDecodeNetworkAddr(t *testing.T) {
for _, tc := range ipAddrTestCases {
d1 := decodeTagData(getReader(tc.binary))
if string(d1) != tc.text {
t.Errorf("decodeNetworkAddr(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), d1, tc.text)
}
}
}
func TestDecodeMACAddr(t *testing.T) {
for _, tc := range macAddrTestCases {
d1 := decodeTagData(getReader(tc.binary))
if string(d1) != tc.text {
t.Errorf("decodeNetworkAddr(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), d1, tc.text)
}
}
}
func TestDecodeIPPrefix(t *testing.T) {
for _, tc := range IPPrefixTestCases {
d1 := decodeTagData(getReader(tc.binary))
if string(d1) != tc.text {
t.Errorf("decodeIPPrefix(0x%s)=%s, want:%s", hex.EncodeToString([]byte(tc.binary)), d1, tc.text)
}
}
}
var compositeCborTestCases = []struct { var compositeCborTestCases = []struct {
binary []byte binary []byte
json string json string
@ -177,12 +174,32 @@ var compositeCborTestCases = []struct {
func TestDecodeCbor2Json(t *testing.T) { func TestDecodeCbor2Json(t *testing.T) {
for _, tc := range compositeCborTestCases { for _, tc := range compositeCborTestCases {
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
_, err := Cbor2JsonManyObjects(tc.binary, buf) err := Cbor2JsonManyObjects(getReader(string(tc.binary)), buf)
if err != nil { if buf.String() != tc.json || err != nil {
t.Errorf("cbor2JsonManyObjects(0x%s) returned error", err) t.Errorf("cbor2JsonManyObjects(0x%s)=%s, want: %s, err:%s", hex.EncodeToString(tc.binary), buf.String(), tc.json, err.Error())
} }
if buf.String() != tc.json { }
t.Errorf("cbor2JsonManyObjects(0x%s)=%s, want: %s", hex.EncodeToString(tc.binary), buf.String(), tc.json) }
var negativeCborTestCases = []struct {
binary []byte
errStr string
}{
{[]byte("\xb9\x64IETF\x20\x65Array\x9f\x20\x00\x18\xc8\x14"), "Tried to Read 18 Bytes.. But hit end of file"},
{[]byte("\xbf\x64IETF\x20\x65Array\x9f\x20\x00\x18\xc8\x14"), "EOF"},
{[]byte("\xbf\x14IETF\x20\x65Array\x9f\x20\x00\x18\xc8\x14"), "Tried to Read 40736 Bytes.. But hit end of file"},
{[]byte("\xbf\x64IETF"), "EOF"},
{[]byte("\xbf\x64IETF\x20\x65Array\x9f\x20\x00\x18\xc8\xff\xff\xff"), "Invalid Additional Type: 31 in decodeSimpleFloat"},
{[]byte("\xbf\x64IETF\x20\x65Array"), "EOF"},
{[]byte("\xbf\x64"), "Tried to Read 4 Bytes.. But hit end of file"},
}
func TestDecodeNegativeCbor2Json(t *testing.T) {
for _, tc := range negativeCborTestCases {
buf := bytes.NewBuffer([]byte{})
err := Cbor2JsonManyObjects(getReader(string(tc.binary)), buf)
if err == nil || err.Error() != tc.errStr {
t.Errorf("Expected error got:%s, want:%s", err, tc.errStr)
} }
} }
} }

55
internal/cbor/examples/genLog.go Normal file
View File

@ -0,0 +1,55 @@
package main
import (
"compress/zlib"
"flag"
"io"
"log"
"os"
"time"
"github.com/rs/zerolog"
)
func writeLog(fname string, count int, useCompress bool) {
opFile := os.Stdout
if fname != "<stdout>" {
fil, _ := os.Create(fname)
opFile = fil
defer func() {
if err := fil.Close(); err != nil {
log.Fatal(err)
}
}()
}
var f io.WriteCloser = opFile
if useCompress {
f = zlib.NewWriter(f)
defer func() {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}()
}
zerolog.TimestampFunc = func() time.Time { return time.Now().Round(time.Second) }
log := zerolog.New(f).With().
Timestamp().
Logger()
for i := 0; i < count; i++ {
log.Error().
Int("Fault", 41650+i).Msg("Some Message")
}
}
func main() {
outFile := flag.String("out", "<stdout>", "Output File to which logs will be written to (WILL overwrite if already present).")
numLogs := flag.Int("num", 10, "Number of log messages to generate.")
doCompress := flag.Bool("compress", false, "Enable inline compressed writer")
flag.Parse()
writeLog(*outFile, *numLogs, *doCompress)
}

10
internal/cbor/examples/makefile Normal file
View File

@ -0,0 +1,10 @@
all: genLogJSON genLogCBOR
genLogJSON: genLog.go
go build -o genLogJSON genLog.go
genLogCBOR: genLog.go
go build -tags binary_log -o genLogCBOR genLog.go
clean:
rm -f genLogJSON genLogCBOR

7
internal/cbor/string.go
View File

@ -48,8 +48,13 @@ func AppendBytes(dst, s []byte) []byte {
func AppendEmbeddedJSON(dst, s []byte) []byte { func AppendEmbeddedJSON(dst, s []byte) []byte {
major := majorTypeTags major := majorTypeTags
minor := additionalTypeEmbeddedJSON minor := additionalTypeEmbeddedJSON
dst = append(dst, byte(major|minor))
// Append the TAG to indicate this is Embedded JSON.
dst = append(dst, byte(major|additionalTypeIntUint16))
dst = append(dst, byte(minor>>8))
dst = append(dst, byte(minor&0xff))
// Append the JSON Object as Byte String.
major = majorTypeByteString major = majorTypeByteString
l := len(s) l := len(s)

42
internal/cbor/types.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"math" "math"
"net"
) )
// AppendNull inserts a 'Nil' object into the dst byte array. // AppendNull inserts a 'Nil' object into the dst byte array.
@ -430,9 +431,40 @@ func AppendArrayDelim(dst []byte) []byte {
return dst return dst
} }
func AppendHex (dst []byte, val []byte) []byte { // AppendIPAddr encodes and inserts an IP Address (IPv4 or IPv6).
dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16)) func AppendIPAddr(dst []byte, ip net.IP) []byte {
dst = append(dst, byte(additionalTypeTagHexString>>8)) dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16))
dst = append(dst, byte(additionalTypeTagHexString&0xff)) dst = append(dst, byte(additionalTypeTagNetworkAddr>>8))
return AppendBytes(dst, val) dst = append(dst, byte(additionalTypeTagNetworkAddr&0xff))
return AppendBytes(dst, ip)
}
// AppendIPPrefix encodes and inserts an IP Address Prefix (Address + Mask Length).
func AppendIPPrefix(dst []byte, pfx net.IPNet) []byte {
dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16))
dst = append(dst, byte(additionalTypeTagNetworkPrefix>>8))
dst = append(dst, byte(additionalTypeTagNetworkPrefix&0xff))
// Prefix is a tuple (aka MAP of 1 pair of elements) -
// first element is prefix, second is mask length.
dst = append(dst, byte(majorTypeMap|0x1))
dst = AppendBytes(dst, pfx.IP)
maskLen, _ := pfx.Mask.Size()
return AppendUint8(dst, uint8(maskLen))
}
// AppendMACAddr encodes and inserts an Hardware (MAC) address.
func AppendMACAddr(dst []byte, ha net.HardwareAddr) []byte {
dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16))
dst = append(dst, byte(additionalTypeTagNetworkAddr>>8))
dst = append(dst, byte(additionalTypeTagNetworkAddr&0xff))
return AppendBytes(dst, ha)
}
// AppendHex adds a TAG and inserts a hex bytes as a string.
func AppendHex(dst []byte, val []byte) []byte {
dst = append(dst, byte(majorTypeTags|additionalTypeIntUint16))
dst = append(dst, byte(additionalTypeTagHexString>>8))
dst = append(dst, byte(additionalTypeTagHexString&0xff))
return AppendBytes(dst, val)
} }

67
internal/cbor/types_test.go
View File

@ -2,6 +2,7 @@ package cbor
import ( import (
"encoding/hex" "encoding/hex"
"net"
"testing" "testing"
) )
@ -182,6 +183,72 @@ func TestAppendFloat32(t *testing.T) {
} }
} }
var ipAddrTestCases = []struct {
ipaddr net.IP
text string // ASCII representation of ipaddr
binary string // CBOR representation of ipaddr
}{
{net.IP{10, 0, 0, 1}, "\"10.0.0.1\"", "\xd9\x01\x04\x44\x0a\x00\x00\x01"},
{net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x0, 0x0, 0x0, 0x0, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34},
"\"2001:db8:85a3::8a2e:370:7334\"",
"\xd9\x01\x04\x50\x20\x01\x0d\xb8\x85\xa3\x00\x00\x00\x00\x8a\x2e\x03\x70\x73\x34"},
}
func TestAppendNetworkAddr(t *testing.T) {
for _, tc := range ipAddrTestCases {
s := AppendIPAddr([]byte{}, tc.ipaddr)
got := string(s)
if got != tc.binary {
t.Errorf("AppendIPAddr(%s)=0x%s, want: 0x%s",
tc.ipaddr, hex.EncodeToString(s),
hex.EncodeToString([]byte(tc.binary)))
}
}
}
var macAddrTestCases = []struct {
macaddr net.HardwareAddr
text string // ASCII representation of macaddr
binary string // CBOR representation of macaddr
}{
{net.HardwareAddr{0x12, 0x34, 0x56, 0x78, 0x90, 0xab}, "\"12:34:56:78:90:ab\"", "\xd9\x01\x04\x46\x12\x34\x56\x78\x90\xab"},
{net.HardwareAddr{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3}, "\"20:01:0d:b8:85:a3\"", "\xd9\x01\x04\x46\x20\x01\x0d\xb8\x85\xa3"},
}
func TestAppendMacAddr(t *testing.T) {
for _, tc := range macAddrTestCases {
s := AppendMACAddr([]byte{}, tc.macaddr)
got := string(s)
if got != tc.binary {
t.Errorf("AppendMACAddr(%s)=0x%s, want: 0x%s",
tc.macaddr.String(), hex.EncodeToString(s),
hex.EncodeToString([]byte(tc.binary)))
}
}
}
var IPPrefixTestCases = []struct {
pfx net.IPNet
text string // ASCII representation of pfx
binary string // CBOR representation of pfx
}{
{net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.CIDRMask(0, 32)}, "\"0.0.0.0/0\"", "\xd9\x01\x05\xa1\x44\x00\x00\x00\x00\x00"},
{net.IPNet{IP: net.IP{192, 168, 0, 100}, Mask: net.CIDRMask(24, 32)}, "\"192.168.0.100/24\"",
"\xd9\x01\x05\xa1\x44\xc0\xa8\x00\x64\x18\x18"},
}
func TestAppendIPPrefix(t *testing.T) {
for _, tc := range IPPrefixTestCases {
s := AppendIPPrefix([]byte{}, tc.pfx)
got := string(s)
if got != tc.binary {
t.Errorf("AppendIPPrefix(%s)=0x%s, want: 0x%s",
tc.pfx.String(), hex.EncodeToString(s),
hex.EncodeToString([]byte(tc.binary)))
}
}
}
func BenchmarkAppendInt(b *testing.B) { func BenchmarkAppendInt(b *testing.B) {
type st struct { type st struct {
sz byte sz byte

19
internal/json/types.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"math" "math"
"net"
"strconv" "strconv"
) )
@ -333,6 +334,8 @@ func AppendInterface(dst []byte, i interface{}) []byte {
return append(dst, marshaled...) return append(dst, marshaled...)
} }
// AppendObjectData takes in an object that is already in a byte array
// and adds it to the dst.
func AppendObjectData(dst []byte, o []byte) []byte { func AppendObjectData(dst []byte, o []byte) []byte {
// Two conditions we want to put a ',' between existing content and // Two conditions we want to put a ',' between existing content and
// new content: // new content:
@ -345,3 +348,19 @@ func AppendObjectData(dst []byte, o []byte) []byte {
} }
return append(dst, o...) return append(dst, o...)
} }
// AppendIPAddr adds IPv4 or IPv6 address to dst.
func AppendIPAddr(dst []byte, ip net.IP) []byte {
return AppendString(dst, ip.String())
}
// AppendIPPrefix adds IPv4 or IPv6 Prefix (address & mask) to dst.
func AppendIPPrefix(dst []byte, pfx net.IPNet) []byte {
return AppendString(dst, pfx.String())
}
// AppendMACAddr adds MAC address to dst.
func AppendMACAddr(dst []byte, ha net.HardwareAddr) []byte {
return AppendString(dst, ha.String())
}

104
internal/json/types_test.go
View File

@ -2,6 +2,7 @@ package json
import ( import (
"math" "math"
"net"
"reflect" "reflect"
"testing" "testing"
) )
@ -61,3 +62,106 @@ func TestAppendType(t *testing.T) {
}) })
} }
} }
func Test_appendMAC(t *testing.T) {
MACtests := []struct {
input string
want []byte
}{
{"01:23:45:67:89:ab", []byte(`"01:23:45:67:89:ab"`)},
{"cd:ef:11:22:33:44", []byte(`"cd:ef:11:22:33:44"`)},
}
for _, tt := range MACtests {
t.Run("MAC", func(t *testing.T) {
ha, _ := net.ParseMAC(tt.input)
if got := AppendMACAddr([]byte{}, ha); !reflect.DeepEqual(got, tt.want) {
t.Errorf("appendMACAddr() = %s, want %s", got, tt.want)
}
})
}
}
func Test_appendIP(t *testing.T) {
IPv4tests := []struct {
input net.IP
want []byte
}{
{net.IP{0, 0, 0, 0}, []byte(`"0.0.0.0"`)},
{net.IP{192, 0, 2, 200}, []byte(`"192.0.2.200"`)},
}
for _, tt := range IPv4tests {
t.Run("IPv4", func(t *testing.T) {
if got := AppendIPAddr([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("appendIPAddr() = %s, want %s", got, tt.want)
}
})
}
IPv6tests := []struct {
input net.IP
want []byte
}{
{net.IPv6zero, []byte(`"::"`)},
{net.IPv6linklocalallnodes, []byte(`"ff02::1"`)},
{net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}, []byte(`"2001:db8:85a3::8a2e:370:7334"`)},
}
for _, tt := range IPv6tests {
t.Run("IPv6", func(t *testing.T) {
if got := AppendIPAddr([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("appendIPAddr() = %s, want %s", got, tt.want)
}
})
}
}
func Test_appendIPPrefix(t *testing.T) {
IPv4Prefixtests := []struct {
input net.IPNet
want []byte
}{
{net.IPNet{IP: net.IP{0, 0, 0, 0}, Mask: net.IPv4Mask(0, 0, 0, 0)}, []byte(`"0.0.0.0/0"`)},
{net.IPNet{IP: net.IP{192, 0, 2, 200}, Mask: net.IPv4Mask(255, 255, 255, 0)}, []byte(`"192.0.2.200/24"`)},
}
for _, tt := range IPv4Prefixtests {
t.Run("IPv4", func(t *testing.T) {
if got := AppendIPPrefix([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("appendIPPrefix() = %s, want %s", got, tt.want)
}
})
}
IPv6Prefixtests := []struct {
input net.IPNet
want []byte
}{
{net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}, []byte(`"::/0"`)},
{net.IPNet{IP: net.IPv6linklocalallnodes, Mask: net.CIDRMask(128, 128)}, []byte(`"ff02::1/128"`)},
{net.IPNet{IP: net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34},
Mask: net.CIDRMask(64, 128)},
[]byte(`"2001:db8:85a3::8a2e:370:7334/64"`)},
}
for _, tt := range IPv6Prefixtests {
t.Run("IPv6", func(t *testing.T) {
if got := AppendIPPrefix([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("appendIPPrefix() = %s, want %s", got, tt.want)
}
})
}
}
func Test_appendMac(t *testing.T) {
MACtests := []struct {
input net.HardwareAddr
want []byte
}{
{net.HardwareAddr{0x12, 0x34, 0x56, 0x78, 0x90, 0xab}, []byte(`"12:34:56:78:90:ab"`)},
{net.HardwareAddr{0x12, 0x34, 0x00, 0x00, 0x90, 0xab}, []byte(`"12:34:00:00:90:ab"`)},
}
for _, tt := range MACtests {
t.Run("MAC", func(t *testing.T) {
if got := AppendMACAddr([]byte{}, tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("appendMAC() = %s, want %s", got, tt.want)
}
})
}
}

34
log_example_test.go
View File

@ -5,6 +5,7 @@ package zerolog_test
import ( import (
"errors" "errors"
stdlog "log" stdlog "log"
"net"
"os" "os"
"time" "time"
@ -395,3 +396,36 @@ func ExampleContext_Durs() {
// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"} // Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
} }
func ExampleContext_IPAddr() {
hostIP := net.IP{192, 168, 0, 100}
log := zerolog.New(os.Stdout).With().
IPAddr("HostIP", hostIP).
Logger()
log.Log().Msg("hello world")
// Output: {"HostIP":"192.168.0.100","message":"hello world"}
}
func ExampleContext_IPPrefix() {
route := net.IPNet{IP: net.IP{192, 168, 0, 0}, Mask: net.CIDRMask(24, 32)}
log := zerolog.New(os.Stdout).With().
IPPrefix("Route", route).
Logger()
log.Log().Msg("hello world")
// Output: {"Route":"192.168.0.0/24","message":"hello world"}
}
func ExampleContext_MacAddr() {
mac := net.HardwareAddr{0x00, 0x14, 0x22, 0x01, 0x23, 0x45}
log := zerolog.New(os.Stdout).With().
MACAddr("hostMAC", mac).
Logger()
log.Log().Msg("hello world")
// Output: {"hostMAC":"00:14:22:01:23:45","message":"hello world"}
}

10
log_test.go
View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"net"
"reflect" "reflect"
"runtime" "runtime"
"testing" "testing"
@ -127,10 +128,11 @@ func TestFieldsMap(t *testing.T) {
"uint64": uint64(10), "uint64": uint64(10),
"float32": float32(11), "float32": float32(11),
"float64": float64(12), "float64": float64(12),
"ipv6": net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34},
"dur": 1 * time.Second, "dur": 1 * time.Second,
"time": time.Time{}, "time": time.Time{},
}).Msg("") }).Msg("")
if got, want := decodeIfBinaryToString(out.Bytes()), `{"bool":true,"bytes":"bar","dur":1000,"error":"some error","float32":11,"float64":12,"int":1,"int16":3,"int32":4,"int64":5,"int8":2,"nil":null,"string":"foo","time":"0001-01-01T00:00:00Z","uint":6,"uint16":8,"uint32":9,"uint64":10,"uint8":7}`+"\n"; got != want { if got, want := decodeIfBinaryToString(out.Bytes()), `{"bool":true,"bytes":"bar","dur":1000,"error":"some error","float32":11,"float64":12,"int":1,"int16":3,"int32":4,"int64":5,"int8":2,"ipv6":"2001:db8:85a3::8a2e:370:7334","nil":null,"string":"foo","time":"0001-01-01T00:00:00Z","uint":6,"uint16":8,"uint32":9,"uint64":10,"uint8":7}`+"\n"; got != want {
t.Errorf("invalid log output:\ngot: %v\nwant: %v", got, want) t.Errorf("invalid log output:\ngot: %v\nwant: %v", got, want)
} }
} }
@ -186,13 +188,17 @@ func TestFields(t *testing.T) {
Uint16("uint16", 8). Uint16("uint16", 8).
Uint32("uint32", 9). Uint32("uint32", 9).
Uint64("uint64", 10). Uint64("uint64", 10).
IPAddr("IPv4", net.IP{192, 168, 0, 100}).
IPAddr("IPv6", net.IP{0x20, 0x01, 0x0d, 0xb8, 0x85, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x2e, 0x03, 0x70, 0x73, 0x34}).
MACAddr("Mac", net.HardwareAddr{0x00, 0x14, 0x22, 0x01, 0x23, 0x45}).
IPPrefix("Prefix", net.IPNet{IP: net.IP{192, 168, 0, 100}, Mask: net.CIDRMask(24, 32)}).
Float32("float32", 11). Float32("float32", 11).
Float64("float64", 12). Float64("float64", 12).
Dur("dur", 1*time.Second). Dur("dur", 1*time.Second).
Time("time", time.Time{}). Time("time", time.Time{}).
TimeDiff("diff", now, now.Add(-10*time.Second)). TimeDiff("diff", now, now.Add(-10*time.Second)).
Msg("") Msg("")
if got, want := decodeIfBinaryToString(out.Bytes()), `{"caller":"`+caller+`","string":"foo","bytes":"bar","hex":"12ef","json":{"some":"json"},"error":"some error","bool":true,"int":1,"int8":2,"int16":3,"int32":4,"int64":5,"uint":6,"uint8":7,"uint16":8,"uint32":9,"uint64":10,"float32":11,"float64":12,"dur":1000,"time":"0001-01-01T00:00:00Z","diff":10000}`+"\n"; got != want { if got, want := decodeIfBinaryToString(out.Bytes()), `{"caller":"`+caller+`","string":"foo","bytes":"bar","hex":"12ef","json":{"some":"json"},"error":"some error","bool":true,"int":1,"int8":2,"int16":3,"int32":4,"int64":5,"uint":6,"uint8":7,"uint16":8,"uint32":9,"uint64":10,"IPv4":"192.168.0.100","IPv6":"2001:db8:85a3::8a2e:370:7334","Mac":"00:14:22:01:23:45","Prefix":"192.168.0.100/24","float32":11,"float64":12,"dur":1000,"time":"0001-01-01T00:00:00Z","diff":10000}`+"\n"; got != want {
t.Errorf("invalid log output:\ngot: %v\nwant: %v", got, want) t.Errorf("invalid log output:\ngot: %v\nwant: %v", got, want)
} }
} }