Browse code

Added SubSecondPrecision flag in fluentd logger

Signed-off-by: dungeonmaster18 <umesh4257@gmail.com>

Umesh Yadav authored on 2017/11/17 03:17:35
Showing 26 changed files
... ...
@@ -48,11 +48,12 @@ const (
48 48
 	defaultRetryWait  = 1000
49 49
 	defaultMaxRetries = math.MaxInt32
50 50
 
51
-	addressKey      = "fluentd-address"
52
-	bufferLimitKey  = "fluentd-buffer-limit"
53
-	retryWaitKey    = "fluentd-retry-wait"
54
-	maxRetriesKey   = "fluentd-max-retries"
55
-	asyncConnectKey = "fluentd-async-connect"
51
+	addressKey            = "fluentd-address"
52
+	bufferLimitKey        = "fluentd-buffer-limit"
53
+	retryWaitKey          = "fluentd-retry-wait"
54
+	maxRetriesKey         = "fluentd-max-retries"
55
+	asyncConnectKey       = "fluentd-async-connect"
56
+	subSecondPrecisionKey = "fluentd-sub-second-precision"
56 57
 )
57 58
 
58 59
 func init() {
... ...
@@ -117,15 +118,23 @@ func New(info logger.Info) (logger.Logger, error) {
117 117
 		}
118 118
 	}
119 119
 
120
+	subSecondPrecision := false
121
+	if info.Config[subSecondPrecisionKey] != "" {
122
+		if subSecondPrecision, err = strconv.ParseBool(info.Config[subSecondPrecisionKey]); err != nil {
123
+			return nil, err
124
+		}
125
+	}
126
+
120 127
 	fluentConfig := fluent.Config{
121
-		FluentPort:       loc.port,
122
-		FluentHost:       loc.host,
123
-		FluentNetwork:    loc.protocol,
124
-		FluentSocketPath: loc.path,
125
-		BufferLimit:      bufferLimit,
126
-		RetryWait:        retryWait,
127
-		MaxRetry:         maxRetries,
128
-		AsyncConnect:     asyncConnect,
128
+		FluentPort:         loc.port,
129
+		FluentHost:         loc.host,
130
+		FluentNetwork:      loc.protocol,
131
+		FluentSocketPath:   loc.path,
132
+		BufferLimit:        bufferLimit,
133
+		RetryWait:          retryWait,
134
+		MaxRetry:           maxRetries,
135
+		AsyncConnect:       asyncConnect,
136
+		SubSecondPrecision: subSecondPrecision,
129 137
 	}
130 138
 
131 139
 	logrus.WithField("container", info.ContainerID).WithField("config", fluentConfig).
... ...
@@ -183,6 +192,7 @@ func ValidateLogOpt(cfg map[string]string) error {
183 183
 		case retryWaitKey:
184 184
 		case maxRetriesKey:
185 185
 		case asyncConnectKey:
186
+		case subSecondPrecisionKey:
186 187
 			// Accepted
187 188
 		default:
188 189
 			return fmt.Errorf("unknown log opt '%s' for fluentd log driver", key)
... ...
@@ -79,10 +79,10 @@ github.com/golang/protobuf 7a211bcf3bce0e3f1d74f9894916e6f116ae83b4
79 79
 # gelf logging driver deps
80 80
 github.com/Graylog2/go-gelf v2
81 81
 
82
-github.com/fluent/fluent-logger-golang v1.2.1
82
+github.com/fluent/fluent-logger-golang v1.3.0
83 83
 # fluent-logger-golang deps
84 84
 github.com/philhofer/fwd 98c11a7a6ec829d672b03833c3d69a7fae1ca972
85
-github.com/tinylib/msgp 75ee40d2601edf122ef667e2a07d600d4c44490c
85
+github.com/tinylib/msgp 3b556c64540842d4f82967be066a7f7fffc3adad
86 86
 
87 87
 # fsnotify
88 88
 github.com/fsnotify/fsnotify v1.4.2
... ...
@@ -21,7 +21,7 @@ import "github.com/fluent/fluent-logger-golang/fluent"
21 21
 
22 22
 GoDoc: http://godoc.org/github.com/fluent/fluent-logger-golang/fluent
23 23
 
24
-##Example
24
+## Example
25 25
 
26 26
 ```go
27 27
 package main
... ...
@@ -44,14 +44,14 @@ func main() {
44 44
     "hoge": "hoge",
45 45
   }
46 46
   error := logger.Post(tag, data)
47
-  // error := logger.Post(tag, time.Time.Now(), data)
47
+  // error := logger.PostWithTime(tag, time.Now(), data)
48 48
   if error != nil {
49 49
     panic(error)
50 50
   }
51 51
 }
52 52
 ```
53 53
 
54
-`data` must be a value like `map[string]literal`, `map[string]interface{}` or `struct`. Logger refers tags `msg` or `codec` of each fields of structs.
54
+`data` must be a value like `map[string]literal`, `map[string]interface{}`, `struct` or [`msgp.Marshaler`](http://godoc.org/github.com/tinylib/msgp/msgp#Marshaler). Logger refers tags `msg` or `codec` of each fields of structs.
55 55
 
56 56
 ## Setting config values
57 57
 
... ...
@@ -59,6 +59,11 @@ func main() {
59 59
 f := fluent.New(fluent.Config{FluentPort: 80, FluentHost: "example.com"})
60 60
 ```
61 61
 
62
+### WriteTimeout
63
+
64
+Sets the timeout for Write call of logger.Post.
65
+Since the default is zero value, Write will not time out.
66
+
62 67
 ## Tests
63 68
 ```
64 69
 go test
... ...
@@ -4,13 +4,14 @@ import (
4 4
 	"encoding/json"
5 5
 	"errors"
6 6
 	"fmt"
7
-	"io"
8 7
 	"math"
9 8
 	"net"
10 9
 	"reflect"
11 10
 	"strconv"
12 11
 	"sync"
13 12
 	"time"
13
+
14
+	"github.com/tinylib/msgp/msgp"
14 15
 )
15 16
 
16 17
 const (
... ...
@@ -19,10 +20,14 @@ const (
19 19
 	defaultSocketPath             = ""
20 20
 	defaultPort                   = 24224
21 21
 	defaultTimeout                = 3 * time.Second
22
+	defaultWriteTimeout           = time.Duration(0) // Write() will not time out
22 23
 	defaultBufferLimit            = 8 * 1024 * 1024
23 24
 	defaultRetryWait              = 500
24 25
 	defaultMaxRetry               = 13
25 26
 	defaultReconnectWaitIncreRate = 1.5
27
+	// Default sub-second precision value to false since it is only compatible
28
+	// with fluentd versions v0.14 and above.
29
+	defaultSubSecondPrecision = false
26 30
 )
27 31
 
28 32
 type Config struct {
... ...
@@ -31,12 +36,17 @@ type Config struct {
31 31
 	FluentNetwork    string        `json:"fluent_network"`
32 32
 	FluentSocketPath string        `json:"fluent_socket_path"`
33 33
 	Timeout          time.Duration `json:"timeout"`
34
+	WriteTimeout     time.Duration `json:"write_timeout"`
34 35
 	BufferLimit      int           `json:"buffer_limit"`
35 36
 	RetryWait        int           `json:"retry_wait"`
36 37
 	MaxRetry         int           `json:"max_retry"`
37 38
 	TagPrefix        string        `json:"tag_prefix"`
38 39
 	AsyncConnect     bool          `json:"async_connect"`
39 40
 	MarshalAsJSON    bool          `json:"marshal_as_json"`
41
+
42
+	// Sub-second precision timestamps are only possible for those using fluentd
43
+	// v0.14+ and serializing their messages with msgpack.
44
+	SubSecondPrecision bool `json:"sub_second_precision"`
40 45
 }
41 46
 
42 47
 type Fluent struct {
... ...
@@ -46,7 +56,7 @@ type Fluent struct {
46 46
 	pending []byte
47 47
 
48 48
 	muconn       sync.Mutex
49
-	conn         io.WriteCloser
49
+	conn         net.Conn
50 50
 	reconnecting bool
51 51
 }
52 52
 
... ...
@@ -67,6 +77,9 @@ func New(config Config) (f *Fluent, err error) {
67 67
 	if config.Timeout == 0 {
68 68
 		config.Timeout = defaultTimeout
69 69
 	}
70
+	if config.WriteTimeout == 0 {
71
+		config.WriteTimeout = defaultWriteTimeout
72
+	}
70 73
 	if config.BufferLimit == 0 {
71 74
 		config.BufferLimit = defaultBufferLimit
72 75
 	}
... ...
@@ -90,9 +103,6 @@ func New(config Config) (f *Fluent, err error) {
90 90
 //
91 91
 // Examples:
92 92
 //
93
-//  // send string
94
-//  f.Post("tag_name", "data")
95
-//
96 93
 //  // send map[string]
97 94
 //  mapStringData := map[string]string{
98 95
 //  	"foo":  "bar",
... ...
@@ -124,6 +134,10 @@ func (f *Fluent) PostWithTime(tag string, tm time.Time, message interface{}) err
124 124
 		tag = f.TagPrefix + "." + tag
125 125
 	}
126 126
 
127
+	if m, ok := message.(msgp.Marshaler); ok {
128
+		return f.EncodeAndPostData(tag, tm, m)
129
+	}
130
+
127 131
 	msg := reflect.ValueOf(message)
128 132
 	msgtype := msg.Type()
129 133
 
... ...
@@ -203,6 +217,9 @@ func (f *Fluent) EncodeData(tag string, tm time.Time, message interface{}) (data
203 203
 		msg := Message{Tag: tag, Time: timeUnix, Record: message}
204 204
 		chunk := &MessageChunk{message: msg}
205 205
 		data, err = json.Marshal(chunk)
206
+	} else if f.Config.SubSecondPrecision {
207
+		msg := &MessageExt{Tag: tag, Time: EventTime(tm), Record: message}
208
+		data, err = msg.MarshalMsg(nil)
206 209
 	} else {
207 210
 		msg := &Message{Tag: tag, Time: timeUnix, Record: message}
208 211
 		data, err = msg.MarshalMsg(nil)
... ...
@@ -297,6 +314,12 @@ func (f *Fluent) send() error {
297 297
 
298 298
 	var err error
299 299
 	if len(f.pending) > 0 {
300
+		t := f.Config.WriteTimeout
301
+		if time.Duration(0) < t {
302
+			f.conn.SetWriteDeadline(time.Now().Add(t))
303
+		} else {
304
+			f.conn.SetWriteDeadline(time.Time{})
305
+		}
300 306
 		_, err = f.conn.Write(f.pending)
301 307
 		if err != nil {
302 308
 			f.conn.Close()
... ...
@@ -2,6 +2,12 @@
2 2
 
3 3
 package fluent
4 4
 
5
+import (
6
+	"time"
7
+
8
+	"github.com/tinylib/msgp/msgp"
9
+)
10
+
5 11
 //msgp:tuple Entry
6 12
 type Entry struct {
7 13
 	Time   int64       `msg:"time"`
... ...
@@ -22,3 +28,69 @@ type Message struct {
22 22
 	Record interface{} `msg:"record"`
23 23
 	Option interface{} `msg:"option"`
24 24
 }
25
+
26
+//msgp:tuple MessageExt
27
+type MessageExt struct {
28
+	Tag    string      `msg:"tag"`
29
+	Time   EventTime   `msg:"time,extension"`
30
+	Record interface{} `msg:"record"`
31
+	Option interface{} `msg:"option"`
32
+}
33
+
34
+// EventTime is an extension to the serialized time value. It builds in support
35
+// for sub-second (nanosecond) precision in serialized timestamps.
36
+//
37
+// You can find the full specification for the msgpack message payload here:
38
+// https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1.
39
+//
40
+// You can find more information on msgpack extension types here:
41
+// https://github.com/tinylib/msgp/wiki/Using-Extensions.
42
+type EventTime time.Time
43
+
44
+const (
45
+	extensionType = 0
46
+	length        = 8
47
+)
48
+
49
+func init() {
50
+	msgp.RegisterExtension(extensionType, func() msgp.Extension { return new(EventTime) })
51
+}
52
+
53
+func (t *EventTime) ExtensionType() int8 { return extensionType }
54
+
55
+func (t *EventTime) Len() int { return length }
56
+
57
+func (t *EventTime) MarshalBinaryTo(b []byte) error {
58
+	// Unwrap to Golang time
59
+	goTime := time.Time(*t)
60
+
61
+	// There's no support for timezones in fluentd's protocol for EventTime.
62
+	// Convert to UTC.
63
+	utc := goTime.UTC()
64
+
65
+	// Warning! Converting seconds to an int32 is a lossy operation. This code
66
+	// will hit the "Year 2038" problem.
67
+	sec := int32(utc.Unix())
68
+	nsec := utc.Nanosecond()
69
+
70
+	// Fill the buffer with 4 bytes for the second component of the timestamp.
71
+	b[0] = byte(sec >> 24)
72
+	b[1] = byte(sec >> 16)
73
+	b[2] = byte(sec >> 8)
74
+	b[3] = byte(sec)
75
+
76
+	// Fill the buffer with 4 bytes for the nanosecond component of the
77
+	// timestamp.
78
+	b[4] = byte(nsec >> 24)
79
+	b[5] = byte(nsec >> 16)
80
+	b[6] = byte(nsec >> 8)
81
+	b[7] = byte(nsec)
82
+
83
+	return nil
84
+}
85
+
86
+// UnmarshalBinary is not implemented since decoding messages is not supported
87
+// by this library.
88
+func (t *EventTime) UnmarshalBinary(b []byte) error {
89
+	return nil
90
+}
... ...
@@ -10,13 +10,13 @@ import (
10 10
 
11 11
 // DecodeMsg implements msgp.Decodable
12 12
 func (z *Entry) DecodeMsg(dc *msgp.Reader) (err error) {
13
-	var ssz uint32
14
-	ssz, err = dc.ReadArrayHeader()
13
+	var zxvk uint32
14
+	zxvk, err = dc.ReadArrayHeader()
15 15
 	if err != nil {
16 16
 		return
17 17
 	}
18
-	if ssz != 2 {
19
-		err = msgp.ArrayError{Wanted: 2, Got: ssz}
18
+	if zxvk != 2 {
19
+		err = msgp.ArrayError{Wanted: 2, Got: zxvk}
20 20
 		return
21 21
 	}
22 22
 	z.Time, err = dc.ReadInt64()
... ...
@@ -32,9 +32,10 @@ func (z *Entry) DecodeMsg(dc *msgp.Reader) (err error) {
32 32
 
33 33
 // EncodeMsg implements msgp.Encodable
34 34
 func (z Entry) EncodeMsg(en *msgp.Writer) (err error) {
35
-	err = en.WriteArrayHeader(2)
35
+	// array header, size 2
36
+	err = en.Append(0x92)
36 37
 	if err != nil {
37
-		return
38
+		return err
38 39
 	}
39 40
 	err = en.WriteInt64(z.Time)
40 41
 	if err != nil {
... ...
@@ -50,7 +51,8 @@ func (z Entry) EncodeMsg(en *msgp.Writer) (err error) {
50 50
 // MarshalMsg implements msgp.Marshaler
51 51
 func (z Entry) MarshalMsg(b []byte) (o []byte, err error) {
52 52
 	o = msgp.Require(b, z.Msgsize())
53
-	o = msgp.AppendArrayHeader(o, 2)
53
+	// array header, size 2
54
+	o = append(o, 0x92)
54 55
 	o = msgp.AppendInt64(o, z.Time)
55 56
 	o, err = msgp.AppendIntf(o, z.Record)
56 57
 	if err != nil {
... ...
@@ -61,16 +63,14 @@ func (z Entry) MarshalMsg(b []byte) (o []byte, err error) {
61 61
 
62 62
 // UnmarshalMsg implements msgp.Unmarshaler
63 63
 func (z *Entry) UnmarshalMsg(bts []byte) (o []byte, err error) {
64
-	{
65
-		var ssz uint32
66
-		ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
67
-		if err != nil {
68
-			return
69
-		}
70
-		if ssz != 2 {
71
-			err = msgp.ArrayError{Wanted: 2, Got: ssz}
72
-			return
73
-		}
64
+	var zbzg uint32
65
+	zbzg, bts, err = msgp.ReadArrayHeaderBytes(bts)
66
+	if err != nil {
67
+		return
68
+	}
69
+	if zbzg != 2 {
70
+		err = msgp.ArrayError{Wanted: 2, Got: zbzg}
71
+		return
74 72
 	}
75 73
 	z.Time, bts, err = msgp.ReadInt64Bytes(bts)
76 74
 	if err != nil {
... ...
@@ -84,51 +84,52 @@ func (z *Entry) UnmarshalMsg(bts []byte) (o []byte, err error) {
84 84
 	return
85 85
 }
86 86
 
87
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
87 88
 func (z Entry) Msgsize() (s int) {
88
-	s = msgp.ArrayHeaderSize + msgp.Int64Size + msgp.GuessSize(z.Record)
89
+	s = 1 + msgp.Int64Size + msgp.GuessSize(z.Record)
89 90
 	return
90 91
 }
91 92
 
92 93
 // DecodeMsg implements msgp.Decodable
93 94
 func (z *Forward) DecodeMsg(dc *msgp.Reader) (err error) {
94
-	var ssz uint32
95
-	ssz, err = dc.ReadArrayHeader()
95
+	var zcmr uint32
96
+	zcmr, err = dc.ReadArrayHeader()
96 97
 	if err != nil {
97 98
 		return
98 99
 	}
99
-	if ssz != 3 {
100
-		err = msgp.ArrayError{Wanted: 3, Got: ssz}
100
+	if zcmr != 3 {
101
+		err = msgp.ArrayError{Wanted: 3, Got: zcmr}
101 102
 		return
102 103
 	}
103 104
 	z.Tag, err = dc.ReadString()
104 105
 	if err != nil {
105 106
 		return
106 107
 	}
107
-	var xsz uint32
108
-	xsz, err = dc.ReadArrayHeader()
108
+	var zajw uint32
109
+	zajw, err = dc.ReadArrayHeader()
109 110
 	if err != nil {
110 111
 		return
111 112
 	}
112
-	if cap(z.Entries) >= int(xsz) {
113
-		z.Entries = z.Entries[:xsz]
113
+	if cap(z.Entries) >= int(zajw) {
114
+		z.Entries = (z.Entries)[:zajw]
114 115
 	} else {
115
-		z.Entries = make([]Entry, xsz)
116
+		z.Entries = make([]Entry, zajw)
116 117
 	}
117
-	for xvk := range z.Entries {
118
-		var ssz uint32
119
-		ssz, err = dc.ReadArrayHeader()
118
+	for zbai := range z.Entries {
119
+		var zwht uint32
120
+		zwht, err = dc.ReadArrayHeader()
120 121
 		if err != nil {
121 122
 			return
122 123
 		}
123
-		if ssz != 2 {
124
-			err = msgp.ArrayError{Wanted: 2, Got: ssz}
124
+		if zwht != 2 {
125
+			err = msgp.ArrayError{Wanted: 2, Got: zwht}
125 126
 			return
126 127
 		}
127
-		z.Entries[xvk].Time, err = dc.ReadInt64()
128
+		z.Entries[zbai].Time, err = dc.ReadInt64()
128 129
 		if err != nil {
129 130
 			return
130 131
 		}
131
-		z.Entries[xvk].Record, err = dc.ReadIntf()
132
+		z.Entries[zbai].Record, err = dc.ReadIntf()
132 133
 		if err != nil {
133 134
 			return
134 135
 		}
... ...
@@ -142,9 +143,10 @@ func (z *Forward) DecodeMsg(dc *msgp.Reader) (err error) {
142 142
 
143 143
 // EncodeMsg implements msgp.Encodable
144 144
 func (z *Forward) EncodeMsg(en *msgp.Writer) (err error) {
145
-	err = en.WriteArrayHeader(3)
145
+	// array header, size 3
146
+	err = en.Append(0x93)
146 147
 	if err != nil {
147
-		return
148
+		return err
148 149
 	}
149 150
 	err = en.WriteString(z.Tag)
150 151
 	if err != nil {
... ...
@@ -154,16 +156,17 @@ func (z *Forward) EncodeMsg(en *msgp.Writer) (err error) {
154 154
 	if err != nil {
155 155
 		return
156 156
 	}
157
-	for xvk := range z.Entries {
158
-		err = en.WriteArrayHeader(2)
157
+	for zbai := range z.Entries {
158
+		// array header, size 2
159
+		err = en.Append(0x92)
159 160
 		if err != nil {
160
-			return
161
+			return err
161 162
 		}
162
-		err = en.WriteInt64(z.Entries[xvk].Time)
163
+		err = en.WriteInt64(z.Entries[zbai].Time)
163 164
 		if err != nil {
164 165
 			return
165 166
 		}
166
-		err = en.WriteIntf(z.Entries[xvk].Record)
167
+		err = en.WriteIntf(z.Entries[zbai].Record)
167 168
 		if err != nil {
168 169
 			return
169 170
 		}
... ...
@@ -178,13 +181,15 @@ func (z *Forward) EncodeMsg(en *msgp.Writer) (err error) {
178 178
 // MarshalMsg implements msgp.Marshaler
179 179
 func (z *Forward) MarshalMsg(b []byte) (o []byte, err error) {
180 180
 	o = msgp.Require(b, z.Msgsize())
181
-	o = msgp.AppendArrayHeader(o, 3)
181
+	// array header, size 3
182
+	o = append(o, 0x93)
182 183
 	o = msgp.AppendString(o, z.Tag)
183 184
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Entries)))
184
-	for xvk := range z.Entries {
185
-		o = msgp.AppendArrayHeader(o, 2)
186
-		o = msgp.AppendInt64(o, z.Entries[xvk].Time)
187
-		o, err = msgp.AppendIntf(o, z.Entries[xvk].Record)
185
+	for zbai := range z.Entries {
186
+		// array header, size 2
187
+		o = append(o, 0x92)
188
+		o = msgp.AppendInt64(o, z.Entries[zbai].Time)
189
+		o, err = msgp.AppendIntf(o, z.Entries[zbai].Record)
188 190
 		if err != nil {
189 191
 			return
190 192
 		}
... ...
@@ -198,48 +203,44 @@ func (z *Forward) MarshalMsg(b []byte) (o []byte, err error) {
198 198
 
199 199
 // UnmarshalMsg implements msgp.Unmarshaler
200 200
 func (z *Forward) UnmarshalMsg(bts []byte) (o []byte, err error) {
201
-	{
202
-		var ssz uint32
203
-		ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
204
-		if err != nil {
205
-			return
206
-		}
207
-		if ssz != 3 {
208
-			err = msgp.ArrayError{Wanted: 3, Got: ssz}
209
-			return
210
-		}
201
+	var zhct uint32
202
+	zhct, bts, err = msgp.ReadArrayHeaderBytes(bts)
203
+	if err != nil {
204
+		return
205
+	}
206
+	if zhct != 3 {
207
+		err = msgp.ArrayError{Wanted: 3, Got: zhct}
208
+		return
211 209
 	}
212 210
 	z.Tag, bts, err = msgp.ReadStringBytes(bts)
213 211
 	if err != nil {
214 212
 		return
215 213
 	}
216
-	var xsz uint32
217
-	xsz, bts, err = msgp.ReadArrayHeaderBytes(bts)
214
+	var zcua uint32
215
+	zcua, bts, err = msgp.ReadArrayHeaderBytes(bts)
218 216
 	if err != nil {
219 217
 		return
220 218
 	}
221
-	if cap(z.Entries) >= int(xsz) {
222
-		z.Entries = z.Entries[:xsz]
219
+	if cap(z.Entries) >= int(zcua) {
220
+		z.Entries = (z.Entries)[:zcua]
223 221
 	} else {
224
-		z.Entries = make([]Entry, xsz)
225
-	}
226
-	for xvk := range z.Entries {
227
-		{
228
-			var ssz uint32
229
-			ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
230
-			if err != nil {
231
-				return
232
-			}
233
-			if ssz != 2 {
234
-				err = msgp.ArrayError{Wanted: 2, Got: ssz}
235
-				return
236
-			}
222
+		z.Entries = make([]Entry, zcua)
223
+	}
224
+	for zbai := range z.Entries {
225
+		var zxhx uint32
226
+		zxhx, bts, err = msgp.ReadArrayHeaderBytes(bts)
227
+		if err != nil {
228
+			return
237 229
 		}
238
-		z.Entries[xvk].Time, bts, err = msgp.ReadInt64Bytes(bts)
230
+		if zxhx != 2 {
231
+			err = msgp.ArrayError{Wanted: 2, Got: zxhx}
232
+			return
233
+		}
234
+		z.Entries[zbai].Time, bts, err = msgp.ReadInt64Bytes(bts)
239 235
 		if err != nil {
240 236
 			return
241 237
 		}
242
-		z.Entries[xvk].Record, bts, err = msgp.ReadIntfBytes(bts)
238
+		z.Entries[zbai].Record, bts, err = msgp.ReadIntfBytes(bts)
243 239
 		if err != nil {
244 240
 			return
245 241
 		}
... ...
@@ -252,10 +253,11 @@ func (z *Forward) UnmarshalMsg(bts []byte) (o []byte, err error) {
252 252
 	return
253 253
 }
254 254
 
255
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
255 256
 func (z *Forward) Msgsize() (s int) {
256
-	s = msgp.ArrayHeaderSize + msgp.StringPrefixSize + len(z.Tag) + msgp.ArrayHeaderSize
257
-	for xvk := range z.Entries {
258
-		s += msgp.ArrayHeaderSize + msgp.Int64Size + msgp.GuessSize(z.Entries[xvk].Record)
257
+	s = 1 + msgp.StringPrefixSize + len(z.Tag) + msgp.ArrayHeaderSize
258
+	for zbai := range z.Entries {
259
+		s += 1 + msgp.Int64Size + msgp.GuessSize(z.Entries[zbai].Record)
259 260
 	}
260 261
 	s += msgp.GuessSize(z.Option)
261 262
 	return
... ...
@@ -263,13 +265,13 @@ func (z *Forward) Msgsize() (s int) {
263 263
 
264 264
 // DecodeMsg implements msgp.Decodable
265 265
 func (z *Message) DecodeMsg(dc *msgp.Reader) (err error) {
266
-	var ssz uint32
267
-	ssz, err = dc.ReadArrayHeader()
266
+	var zlqf uint32
267
+	zlqf, err = dc.ReadArrayHeader()
268 268
 	if err != nil {
269 269
 		return
270 270
 	}
271
-	if ssz != 4 {
272
-		err = msgp.ArrayError{Wanted: 4, Got: ssz}
271
+	if zlqf != 4 {
272
+		err = msgp.ArrayError{Wanted: 4, Got: zlqf}
273 273
 		return
274 274
 	}
275 275
 	z.Tag, err = dc.ReadString()
... ...
@@ -293,9 +295,10 @@ func (z *Message) DecodeMsg(dc *msgp.Reader) (err error) {
293 293
 
294 294
 // EncodeMsg implements msgp.Encodable
295 295
 func (z *Message) EncodeMsg(en *msgp.Writer) (err error) {
296
-	err = en.WriteArrayHeader(4)
296
+	// array header, size 4
297
+	err = en.Append(0x94)
297 298
 	if err != nil {
298
-		return
299
+		return err
299 300
 	}
300 301
 	err = en.WriteString(z.Tag)
301 302
 	if err != nil {
... ...
@@ -319,7 +322,8 @@ func (z *Message) EncodeMsg(en *msgp.Writer) (err error) {
319 319
 // MarshalMsg implements msgp.Marshaler
320 320
 func (z *Message) MarshalMsg(b []byte) (o []byte, err error) {
321 321
 	o = msgp.Require(b, z.Msgsize())
322
-	o = msgp.AppendArrayHeader(o, 4)
322
+	// array header, size 4
323
+	o = append(o, 0x94)
323 324
 	o = msgp.AppendString(o, z.Tag)
324 325
 	o = msgp.AppendInt64(o, z.Time)
325 326
 	o, err = msgp.AppendIntf(o, z.Record)
... ...
@@ -335,16 +339,14 @@ func (z *Message) MarshalMsg(b []byte) (o []byte, err error) {
335 335
 
336 336
 // UnmarshalMsg implements msgp.Unmarshaler
337 337
 func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) {
338
-	{
339
-		var ssz uint32
340
-		ssz, bts, err = msgp.ReadArrayHeaderBytes(bts)
341
-		if err != nil {
342
-			return
343
-		}
344
-		if ssz != 4 {
345
-			err = msgp.ArrayError{Wanted: 4, Got: ssz}
346
-			return
347
-		}
338
+	var zdaf uint32
339
+	zdaf, bts, err = msgp.ReadArrayHeaderBytes(bts)
340
+	if err != nil {
341
+		return
342
+	}
343
+	if zdaf != 4 {
344
+		err = msgp.ArrayError{Wanted: 4, Got: zdaf}
345
+		return
348 346
 	}
349 347
 	z.Tag, bts, err = msgp.ReadStringBytes(bts)
350 348
 	if err != nil {
... ...
@@ -366,7 +368,122 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) {
366 366
 	return
367 367
 }
368 368
 
369
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
369 370
 func (z *Message) Msgsize() (s int) {
370
-	s = msgp.ArrayHeaderSize + msgp.StringPrefixSize + len(z.Tag) + msgp.Int64Size + msgp.GuessSize(z.Record) + msgp.GuessSize(z.Option)
371
+	s = 1 + msgp.StringPrefixSize + len(z.Tag) + msgp.Int64Size + msgp.GuessSize(z.Record) + msgp.GuessSize(z.Option)
372
+	return
373
+}
374
+
375
+// DecodeMsg implements msgp.Decodable
376
+func (z *MessageExt) DecodeMsg(dc *msgp.Reader) (err error) {
377
+	var zpks uint32
378
+	zpks, err = dc.ReadArrayHeader()
379
+	if err != nil {
380
+		return
381
+	}
382
+	if zpks != 4 {
383
+		err = msgp.ArrayError{Wanted: 4, Got: zpks}
384
+		return
385
+	}
386
+	z.Tag, err = dc.ReadString()
387
+	if err != nil {
388
+		return
389
+	}
390
+	err = dc.ReadExtension(&z.Time)
391
+	if err != nil {
392
+		return
393
+	}
394
+	z.Record, err = dc.ReadIntf()
395
+	if err != nil {
396
+		return
397
+	}
398
+	z.Option, err = dc.ReadIntf()
399
+	if err != nil {
400
+		return
401
+	}
402
+	return
403
+}
404
+
405
+// EncodeMsg implements msgp.Encodable
406
+func (z *MessageExt) EncodeMsg(en *msgp.Writer) (err error) {
407
+	// array header, size 4
408
+	err = en.Append(0x94)
409
+	if err != nil {
410
+		return err
411
+	}
412
+	err = en.WriteString(z.Tag)
413
+	if err != nil {
414
+		return
415
+	}
416
+	err = en.WriteExtension(&z.Time)
417
+	if err != nil {
418
+		return
419
+	}
420
+	err = en.WriteIntf(z.Record)
421
+	if err != nil {
422
+		return
423
+	}
424
+	err = en.WriteIntf(z.Option)
425
+	if err != nil {
426
+		return
427
+	}
428
+	return
429
+}
430
+
431
+// MarshalMsg implements msgp.Marshaler
432
+func (z *MessageExt) MarshalMsg(b []byte) (o []byte, err error) {
433
+	o = msgp.Require(b, z.Msgsize())
434
+	// array header, size 4
435
+	o = append(o, 0x94)
436
+	o = msgp.AppendString(o, z.Tag)
437
+	o, err = msgp.AppendExtension(o, &z.Time)
438
+	if err != nil {
439
+		return
440
+	}
441
+	o, err = msgp.AppendIntf(o, z.Record)
442
+	if err != nil {
443
+		return
444
+	}
445
+	o, err = msgp.AppendIntf(o, z.Option)
446
+	if err != nil {
447
+		return
448
+	}
449
+	return
450
+}
451
+
452
+// UnmarshalMsg implements msgp.Unmarshaler
453
+func (z *MessageExt) UnmarshalMsg(bts []byte) (o []byte, err error) {
454
+	var zjfb uint32
455
+	zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts)
456
+	if err != nil {
457
+		return
458
+	}
459
+	if zjfb != 4 {
460
+		err = msgp.ArrayError{Wanted: 4, Got: zjfb}
461
+		return
462
+	}
463
+	z.Tag, bts, err = msgp.ReadStringBytes(bts)
464
+	if err != nil {
465
+		return
466
+	}
467
+	bts, err = msgp.ReadExtensionBytes(bts, &z.Time)
468
+	if err != nil {
469
+		return
470
+	}
471
+	z.Record, bts, err = msgp.ReadIntfBytes(bts)
472
+	if err != nil {
473
+		return
474
+	}
475
+	z.Option, bts, err = msgp.ReadIntfBytes(bts)
476
+	if err != nil {
477
+		return
478
+	}
479
+	o = bts
480
+	return
481
+}
482
+
483
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
484
+func (z *MessageExt) Msgsize() (s int) {
485
+	s = 1 + msgp.StringPrefixSize + len(z.Tag) + msgp.ExtensionPrefixSize + z.Time.Len() + msgp.GuessSize(z.Record) + msgp.GuessSize(z.Option)
371 486
 	return
372 487
 }
373 488
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+package fluent
1
+
2
+//go:generate msgp
3
+type TestMessage struct {
4
+	Foo  string `msg:"foo" json:"foo,omitempty"`
5
+	Hoge string `msg:"hoge" json:"hoge,omitempty"`
6
+}
0 7
new file mode 100644
... ...
@@ -0,0 +1,125 @@
0
+package fluent
1
+
2
+// NOTE: THIS FILE WAS PRODUCED BY THE
3
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
4
+// DO NOT EDIT
5
+
6
+import (
7
+	"github.com/tinylib/msgp/msgp"
8
+)
9
+
10
+// DecodeMsg implements msgp.Decodable
11
+func (z *TestMessage) DecodeMsg(dc *msgp.Reader) (err error) {
12
+	var field []byte
13
+	_ = field
14
+	var zxvk uint32
15
+	zxvk, err = dc.ReadMapHeader()
16
+	if err != nil {
17
+		return
18
+	}
19
+	for zxvk > 0 {
20
+		zxvk--
21
+		field, err = dc.ReadMapKeyPtr()
22
+		if err != nil {
23
+			return
24
+		}
25
+		switch msgp.UnsafeString(field) {
26
+		case "foo":
27
+			z.Foo, err = dc.ReadString()
28
+			if err != nil {
29
+				return
30
+			}
31
+		case "hoge":
32
+			z.Hoge, err = dc.ReadString()
33
+			if err != nil {
34
+				return
35
+			}
36
+		default:
37
+			err = dc.Skip()
38
+			if err != nil {
39
+				return
40
+			}
41
+		}
42
+	}
43
+	return
44
+}
45
+
46
+// EncodeMsg implements msgp.Encodable
47
+func (z TestMessage) EncodeMsg(en *msgp.Writer) (err error) {
48
+	// map header, size 2
49
+	// write "foo"
50
+	err = en.Append(0x82, 0xa3, 0x66, 0x6f, 0x6f)
51
+	if err != nil {
52
+		return err
53
+	}
54
+	err = en.WriteString(z.Foo)
55
+	if err != nil {
56
+		return
57
+	}
58
+	// write "hoge"
59
+	err = en.Append(0xa4, 0x68, 0x6f, 0x67, 0x65)
60
+	if err != nil {
61
+		return err
62
+	}
63
+	err = en.WriteString(z.Hoge)
64
+	if err != nil {
65
+		return
66
+	}
67
+	return
68
+}
69
+
70
+// MarshalMsg implements msgp.Marshaler
71
+func (z TestMessage) MarshalMsg(b []byte) (o []byte, err error) {
72
+	o = msgp.Require(b, z.Msgsize())
73
+	// map header, size 2
74
+	// string "foo"
75
+	o = append(o, 0x82, 0xa3, 0x66, 0x6f, 0x6f)
76
+	o = msgp.AppendString(o, z.Foo)
77
+	// string "hoge"
78
+	o = append(o, 0xa4, 0x68, 0x6f, 0x67, 0x65)
79
+	o = msgp.AppendString(o, z.Hoge)
80
+	return
81
+}
82
+
83
+// UnmarshalMsg implements msgp.Unmarshaler
84
+func (z *TestMessage) UnmarshalMsg(bts []byte) (o []byte, err error) {
85
+	var field []byte
86
+	_ = field
87
+	var zbzg uint32
88
+	zbzg, bts, err = msgp.ReadMapHeaderBytes(bts)
89
+	if err != nil {
90
+		return
91
+	}
92
+	for zbzg > 0 {
93
+		zbzg--
94
+		field, bts, err = msgp.ReadMapKeyZC(bts)
95
+		if err != nil {
96
+			return
97
+		}
98
+		switch msgp.UnsafeString(field) {
99
+		case "foo":
100
+			z.Foo, bts, err = msgp.ReadStringBytes(bts)
101
+			if err != nil {
102
+				return
103
+			}
104
+		case "hoge":
105
+			z.Hoge, bts, err = msgp.ReadStringBytes(bts)
106
+			if err != nil {
107
+				return
108
+			}
109
+		default:
110
+			bts, err = msgp.Skip(bts)
111
+			if err != nil {
112
+				return
113
+			}
114
+		}
115
+	}
116
+	o = bts
117
+	return
118
+}
119
+
120
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
121
+func (z TestMessage) Msgsize() (s int) {
122
+	s = 1 + 4 + msgp.StringPrefixSize + len(z.Foo) + 5 + msgp.StringPrefixSize + len(z.Hoge)
123
+	return
124
+}
... ...
@@ -1,3 +1,3 @@
1 1
 package fluent
2 2
 
3
-const Version = "1.2.1"
3
+const Version = "1.3.0"
... ...
@@ -1,15 +1,12 @@
1 1
 MessagePack Code Generator [![Build Status](https://travis-ci.org/tinylib/msgp.svg?branch=master)](https://travis-ci.org/tinylib/msgp)
2 2
 =======
3 3
 
4
-[![forthebadge](http://forthebadge.com/badges/uses-badges.svg)](http://forthebadge.com)
5
-[![forthebadge](http://forthebadge.com/badges/ages-12.svg)](http://forthebadge.com)
6
-
7
-This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). It is targeted at the `go generate` [tool](http://tip.golang.org/cmd/go/#hdr-Generate_Go_files_by_processing_source). You can read more about MessagePack [in the wiki](http://github.com/tinylib/msgp/wiki), or at [msgpack.org](http://msgpack.org).
4
+This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). You can read more about MessagePack [in the wiki](http://github.com/tinylib/msgp/wiki), or at [msgpack.org](http://msgpack.org).
8 5
 
9 6
 ### Why?
10 7
 
11 8
 - Use Go as your schema language
12
-- Speeeeeed (400MB/s on modern hardware)
9
+- Performance
13 10
 - [JSON interop](http://godoc.org/github.com/tinylib/msgp/msgp#CopyToJSON)
14 11
 - [User-defined extensions](http://github.com/tinylib/msgp/wiki/Using-Extensions)
15 12
 - Type safety
... ...
@@ -17,8 +14,6 @@ This is a code generation tool and serialization library for [MessagePack](http:
17 17
 
18 18
 ### Quickstart
19 19
 
20
-Note: you need at least go 1.3 to compile this package, and at least go 1.4 to use `go generate`.
21
-
22 20
 In a source file, include the following directive:
23 21
 
24 22
 ```go
... ...
@@ -45,7 +40,7 @@ type Person struct {
45 45
 
46 46
 By default, the code generator will satisfy `msgp.Sizer`, `msgp.Encodable`, `msgp.Decodable`, 
47 47
 `msgp.Marshaler`, and `msgp.Unmarshaler`. Carefully-designed applications can use these methods to do
48
-marshalling/unmarshalling with zero allocations.
48
+marshalling/unmarshalling with zero heap allocations.
49 49
 
50 50
 While `msgp.Marshaler` and `msgp.Unmarshaler` are quite similar to the standard library's
51 51
 `json.Marshaler` and `json.Unmarshaler`, `msgp.Encodable` and `msgp.Decodable` are useful for 
... ...
@@ -62,6 +57,7 @@ of `*bufio.Writer` and `*bufio.Reader`, respectively.)
62 62
  - Generation of both `[]byte`-oriented and `io.Reader/io.Writer`-oriented methods
63 63
  - Support for arbitrary type system extensions
64 64
  - [Preprocessor directives](http://github.com/tinylib/msgp/wiki/Preprocessor-Directives)
65
+ - File-based dependency model means fast codegen regardless of source tree size.
65 66
 
66 67
 Consider the following:
67 68
 ```go
... ...
@@ -84,21 +80,23 @@ the data "type" (`int8`) and the raw binary. You [can see a worked example in th
84 84
 
85 85
 ### Status
86 86
 
87
-Alpha. I _will_ break stuff. There is an open milestone for Beta stability (targeted for January.) Only the `/msgp` sub-directory will have a stability guarantee.
87
+Mostly stable, in that no breaking changes have been made to the `/msgp` library in more than a year. Newer versions
88
+of the code may generate different code than older versions for performance reasons. I (@philhofer) am aware of a
89
+number of stability-critical commercial applications that use this code with good results. But, caveat emptor.
88 90
 
89 91
 You can read more about how `msgp` maps MessagePack types onto Go types [in the wiki](http://github.com/tinylib/msgp/wiki).
90 92
 
91 93
 Here some of the known limitations/restrictions:
92 94
 
93
- - Identifiers from outside the processed source file are assumed (optimistically) to satisfy the generator's interfaces. If this isn't the case, your code will fail to compile.
94
- - Like most serializers, `chan` and `func` fields are ignored, as well as non-exported fields.
95
- - Encoding of `interface{}` is limited to built-ins or types that have explicit encoding methods.
96
- - _Maps must have `string` keys._ This is intentional (as it preserves JSON interop.) Although non-string map keys are not forbidden by the MessagePack standard, many serializers impose this restriction. (It also means *any* well-formed `struct` can be de-serialized into a `map[string]interface{}`.) The only exception to this rule is that the deserializers will allow you to read map keys encoded as `bin` types, due to the fact that some legacy encodings permitted this. (However, those values will still be cast to Go `string`s, and they will be converted to `str` types when re-encoded. It is the responsibility of the user to ensure that map keys are UTF-8 safe in this case.) The same rules hold true for JSON translation.
95
+- Identifiers from outside the processed source file are assumed (optimistically) to satisfy the generator's interfaces. If this isn't the case, your code will fail to compile.
96
+- Like most serializers, `chan` and `func` fields are ignored, as well as non-exported fields.
97
+- Encoding of `interface{}` is limited to built-ins or types that have explicit encoding methods.
98
+- _Maps must have `string` keys._ This is intentional (as it preserves JSON interop.) Although non-string map keys are not forbidden by the MessagePack standard, many serializers impose this restriction. (It also means *any* well-formed `struct` can be de-serialized into a `map[string]interface{}`.) The only exception to this rule is that the deserializers will allow you to read map keys encoded as `bin` types, due to the fact that some legacy encodings permitted this. (However, those values will still be cast to Go `string`s, and they will be converted to `str` types when re-encoded. It is the responsibility of the user to ensure that map keys are UTF-8 safe in this case.) The same rules hold true for JSON translation.
97 99
 
98 100
 If the output compiles, then there's a pretty good chance things are fine. (Plus, we generate tests for you.) *Please, please, please* file an issue if you think the generator is writing broken code.
99 101
 
100 102
 ### Performance
101 103
 
102
-If you like benchmarks, see [here.](https://github.com/alecthomas/go_serialization_benchmarks)
104
+If you like benchmarks, see [here](http://bravenewgeek.com/so-you-wanna-go-fast/) and [here](https://github.com/alecthomas/go_serialization_benchmarks).
103 105
 
104
-As one might expect, the generated methods that deal with `[]byte` are faster, but the `io.Reader/Writer` methods are generally more memory-efficient for large (> 2KB) objects.
106
+As one might expect, the generated methods that deal with `[]byte` are faster for small objects, but the `io.Reader/Writer` methods are generally more memory-efficient (and, at some point, faster) for large (> 2KB) objects.
105 107
new file mode 100644
... ...
@@ -0,0 +1,24 @@
0
+// +build linux,!appengine
1
+
2
+package msgp
3
+
4
+import (
5
+	"os"
6
+	"syscall"
7
+)
8
+
9
+func adviseRead(mem []byte) {
10
+	syscall.Madvise(mem, syscall.MADV_SEQUENTIAL|syscall.MADV_WILLNEED)
11
+}
12
+
13
+func adviseWrite(mem []byte) {
14
+	syscall.Madvise(mem, syscall.MADV_SEQUENTIAL)
15
+}
16
+
17
+func fallocate(f *os.File, sz int64) error {
18
+	err := syscall.Fallocate(int(f.Fd()), 0, 0, sz)
19
+	if err == syscall.ENOTSUP {
20
+		return f.Truncate(sz)
21
+	}
22
+	return err
23
+}
0 24
new file mode 100644
... ...
@@ -0,0 +1,17 @@
0
+// +build !linux appengine
1
+
2
+package msgp
3
+
4
+import (
5
+	"os"
6
+)
7
+
8
+// TODO: darwin, BSD support
9
+
10
+func adviseRead(mem []byte) {}
11
+
12
+func adviseWrite(mem []byte) {}
13
+
14
+func fallocate(f *os.File, sz int64) error {
15
+	return f.Truncate(sz)
16
+}
0 17
new file mode 100644
... ...
@@ -0,0 +1,15 @@
0
+// +build appengine
1
+
2
+package msgp
3
+
4
+// let's just assume appengine
5
+// uses 64-bit hardware...
6
+const smallint = false
7
+
8
+func UnsafeString(b []byte) string {
9
+	return string(b)
10
+}
11
+
12
+func UnsafeBytes(s string) []byte {
13
+	return []byte(s)
14
+}
... ...
@@ -1,20 +1,21 @@
1 1
 package msgp
2 2
 
3
-import (
4
-	"testing"
5
-)
3
+type timer interface {
4
+	StartTimer()
5
+	StopTimer()
6
+}
6 7
 
7 8
 // EndlessReader is an io.Reader
8 9
 // that loops over the same data
9 10
 // endlessly. It is used for benchmarking.
10 11
 type EndlessReader struct {
11
-	tb     *testing.B
12
+	tb     timer
12 13
 	data   []byte
13 14
 	offset int
14 15
 }
15 16
 
16 17
 // NewEndlessReader returns a new endless reader
17
-func NewEndlessReader(b []byte, tb *testing.B) *EndlessReader {
18
+func NewEndlessReader(b []byte, tb timer) *EndlessReader {
18 19
 	return &EndlessReader{tb: tb, data: b, offset: 0}
19 20
 }
20 21
 
... ...
@@ -226,7 +226,7 @@ func (mw *Writer) WriteExtension(e Extension) error {
226 226
 // peek at the extension type, assuming the next
227 227
 // kind to be read is Extension
228 228
 func (m *Reader) peekExtensionType() (int8, error) {
229
-	p, err := m.r.Peek(2)
229
+	p, err := m.R.Peek(2)
230 230
 	if err != nil {
231 231
 		return 0, err
232 232
 	}
... ...
@@ -238,7 +238,7 @@ func (m *Reader) peekExtensionType() (int8, error) {
238 238
 		return int8(p[1]), nil
239 239
 	}
240 240
 	size := spec.size
241
-	p, err = m.r.Peek(int(size))
241
+	p, err = m.R.Peek(int(size))
242 242
 	if err != nil {
243 243
 		return 0, err
244 244
 	}
... ...
@@ -273,7 +273,7 @@ func peekExtension(b []byte) (int8, error) {
273 273
 // e.Type() is not the same as the wire type.
274 274
 func (m *Reader) ReadExtension(e Extension) (err error) {
275 275
 	var p []byte
276
-	p, err = m.r.Peek(2)
276
+	p, err = m.R.Peek(2)
277 277
 	if err != nil {
278 278
 		return
279 279
 	}
... ...
@@ -286,13 +286,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
286 286
 			err = errExt(int8(p[1]), e.ExtensionType())
287 287
 			return
288 288
 		}
289
-		p, err = m.r.Peek(3)
289
+		p, err = m.R.Peek(3)
290 290
 		if err != nil {
291 291
 			return
292 292
 		}
293 293
 		err = e.UnmarshalBinary(p[2:])
294 294
 		if err == nil {
295
-			_, err = m.r.Skip(3)
295
+			_, err = m.R.Skip(3)
296 296
 		}
297 297
 		return
298 298
 
... ...
@@ -301,13 +301,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
301 301
 			err = errExt(int8(p[1]), e.ExtensionType())
302 302
 			return
303 303
 		}
304
-		p, err = m.r.Peek(4)
304
+		p, err = m.R.Peek(4)
305 305
 		if err != nil {
306 306
 			return
307 307
 		}
308 308
 		err = e.UnmarshalBinary(p[2:])
309 309
 		if err == nil {
310
-			_, err = m.r.Skip(4)
310
+			_, err = m.R.Skip(4)
311 311
 		}
312 312
 		return
313 313
 
... ...
@@ -316,13 +316,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
316 316
 			err = errExt(int8(p[1]), e.ExtensionType())
317 317
 			return
318 318
 		}
319
-		p, err = m.r.Peek(6)
319
+		p, err = m.R.Peek(6)
320 320
 		if err != nil {
321 321
 			return
322 322
 		}
323 323
 		err = e.UnmarshalBinary(p[2:])
324 324
 		if err == nil {
325
-			_, err = m.r.Skip(6)
325
+			_, err = m.R.Skip(6)
326 326
 		}
327 327
 		return
328 328
 
... ...
@@ -331,13 +331,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
331 331
 			err = errExt(int8(p[1]), e.ExtensionType())
332 332
 			return
333 333
 		}
334
-		p, err = m.r.Peek(10)
334
+		p, err = m.R.Peek(10)
335 335
 		if err != nil {
336 336
 			return
337 337
 		}
338 338
 		err = e.UnmarshalBinary(p[2:])
339 339
 		if err == nil {
340
-			_, err = m.r.Skip(10)
340
+			_, err = m.R.Skip(10)
341 341
 		}
342 342
 		return
343 343
 
... ...
@@ -346,18 +346,18 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
346 346
 			err = errExt(int8(p[1]), e.ExtensionType())
347 347
 			return
348 348
 		}
349
-		p, err = m.r.Peek(18)
349
+		p, err = m.R.Peek(18)
350 350
 		if err != nil {
351 351
 			return
352 352
 		}
353 353
 		err = e.UnmarshalBinary(p[2:])
354 354
 		if err == nil {
355
-			_, err = m.r.Skip(18)
355
+			_, err = m.R.Skip(18)
356 356
 		}
357 357
 		return
358 358
 
359 359
 	case mext8:
360
-		p, err = m.r.Peek(3)
360
+		p, err = m.R.Peek(3)
361 361
 		if err != nil {
362 362
 			return
363 363
 		}
... ...
@@ -369,7 +369,7 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
369 369
 		off = 3
370 370
 
371 371
 	case mext16:
372
-		p, err = m.r.Peek(4)
372
+		p, err = m.R.Peek(4)
373 373
 		if err != nil {
374 374
 			return
375 375
 		}
... ...
@@ -381,7 +381,7 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
381 381
 		off = 4
382 382
 
383 383
 	case mext32:
384
-		p, err = m.r.Peek(6)
384
+		p, err = m.R.Peek(6)
385 385
 		if err != nil {
386 386
 			return
387 387
 		}
... ...
@@ -397,13 +397,13 @@ func (m *Reader) ReadExtension(e Extension) (err error) {
397 397
 		return
398 398
 	}
399 399
 
400
-	p, err = m.r.Peek(read + off)
400
+	p, err = m.R.Peek(read + off)
401 401
 	if err != nil {
402 402
 		return
403 403
 	}
404 404
 	err = e.UnmarshalBinary(p[off:])
405 405
 	if err == nil {
406
-		_, err = m.r.Skip(read + off)
406
+		_, err = m.R.Skip(read + off)
407 407
 	}
408 408
 	return
409 409
 }
410 410
new file mode 100644
... ...
@@ -0,0 +1,92 @@
0
+// +build linux darwin dragonfly freebsd netbsd openbsd
1
+// +build !appengine
2
+
3
+package msgp
4
+
5
+import (
6
+	"os"
7
+	"syscall"
8
+)
9
+
10
+// ReadFile reads a file into 'dst' using
11
+// a read-only memory mapping. Consequently,
12
+// the file must be mmap-able, and the
13
+// Unmarshaler should never write to
14
+// the source memory. (Methods generated
15
+// by the msgp tool obey that constraint, but
16
+// user-defined implementations may not.)
17
+//
18
+// Reading and writing through file mappings
19
+// is only efficient for large files; small
20
+// files are best read and written using
21
+// the ordinary streaming interfaces.
22
+//
23
+func ReadFile(dst Unmarshaler, file *os.File) error {
24
+	stat, err := file.Stat()
25
+	if err != nil {
26
+		return err
27
+	}
28
+	data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
29
+	if err != nil {
30
+		return err
31
+	}
32
+	adviseRead(data)
33
+	_, err = dst.UnmarshalMsg(data)
34
+	uerr := syscall.Munmap(data)
35
+	if err == nil {
36
+		err = uerr
37
+	}
38
+	return err
39
+}
40
+
41
+// MarshalSizer is the combination
42
+// of the Marshaler and Sizer
43
+// interfaces.
44
+type MarshalSizer interface {
45
+	Marshaler
46
+	Sizer
47
+}
48
+
49
+// WriteFile writes a file from 'src' using
50
+// memory mapping. It overwrites the entire
51
+// contents of the previous file.
52
+// The mapping size is calculated
53
+// using the `Msgsize()` method
54
+// of 'src', so it must produce a result
55
+// equal to or greater than the actual encoded
56
+// size of the object. Otherwise,
57
+// a fault (SIGBUS) will occur.
58
+//
59
+// Reading and writing through file mappings
60
+// is only efficient for large files; small
61
+// files are best read and written using
62
+// the ordinary streaming interfaces.
63
+//
64
+// NOTE: The performance of this call
65
+// is highly OS- and filesystem-dependent.
66
+// Users should take care to test that this
67
+// performs as expected in a production environment.
68
+// (Linux users should run a kernel and filesystem
69
+// that support fallocate(2) for the best results.)
70
+func WriteFile(src MarshalSizer, file *os.File) error {
71
+	sz := src.Msgsize()
72
+	err := fallocate(file, int64(sz))
73
+	if err != nil {
74
+		return err
75
+	}
76
+	data, err := syscall.Mmap(int(file.Fd()), 0, sz, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
77
+	if err != nil {
78
+		return err
79
+	}
80
+	adviseWrite(data)
81
+	chunk := data[:0]
82
+	chunk, err = src.MarshalMsg(chunk)
83
+	if err != nil {
84
+		return err
85
+	}
86
+	uerr := syscall.Munmap(data)
87
+	if uerr != nil {
88
+		return uerr
89
+	}
90
+	return file.Truncate(int64(len(chunk)))
91
+}
0 92
new file mode 100644
... ...
@@ -0,0 +1,47 @@
0
+// +build windows appengine
1
+
2
+package msgp
3
+
4
+import (
5
+	"io/ioutil"
6
+	"os"
7
+)
8
+
9
+// MarshalSizer is the combination
10
+// of the Marshaler and Sizer
11
+// interfaces.
12
+type MarshalSizer interface {
13
+	Marshaler
14
+	Sizer
15
+}
16
+
17
+func ReadFile(dst Unmarshaler, file *os.File) error {
18
+	if u, ok := dst.(Decodable); ok {
19
+		return u.DecodeMsg(NewReader(file))
20
+	}
21
+
22
+	data, err := ioutil.ReadAll(file)
23
+	if err != nil {
24
+		return err
25
+	}
26
+	_, err = dst.UnmarshalMsg(data)
27
+	return err
28
+}
29
+
30
+func WriteFile(src MarshalSizer, file *os.File) error {
31
+	if e, ok := src.(Encodable); ok {
32
+		w := NewWriter(file)
33
+		err := e.EncodeMsg(w)
34
+		if err == nil {
35
+			err = w.Flush()
36
+		}
37
+		return err
38
+	}
39
+
40
+	raw, err := src.MarshalMsg(nil)
41
+	if err != nil {
42
+		return err
43
+	}
44
+	_, err = file.Write(raw)
45
+	return err
46
+}
... ...
@@ -66,7 +66,7 @@ func (r *Reader) WriteToJSON(w io.Writer) (n int64, err error) {
66 66
 	if jsw, ok := w.(jsWriter); ok {
67 67
 		j = jsw
68 68
 	} else {
69
-		bf = bufio.NewWriterSize(w, 512)
69
+		bf = bufio.NewWriter(w)
70 70
 		j = bf
71 71
 	}
72 72
 	var nn int
... ...
@@ -333,7 +333,7 @@ func rwExtension(dst jsWriter, src *Reader) (n int, err error) {
333 333
 
334 334
 func rwString(dst jsWriter, src *Reader) (n int, err error) {
335 335
 	var p []byte
336
-	p, err = src.r.Peek(1)
336
+	p, err = src.R.Peek(1)
337 337
 	if err != nil {
338 338
 		return
339 339
 	}
... ...
@@ -342,25 +342,25 @@ func rwString(dst jsWriter, src *Reader) (n int, err error) {
342 342
 
343 343
 	if isfixstr(lead) {
344 344
 		read = int(rfixstr(lead))
345
-		src.r.Skip(1)
345
+		src.R.Skip(1)
346 346
 		goto write
347 347
 	}
348 348
 
349 349
 	switch lead {
350 350
 	case mstr8:
351
-		p, err = src.r.Next(2)
351
+		p, err = src.R.Next(2)
352 352
 		if err != nil {
353 353
 			return
354 354
 		}
355 355
 		read = int(uint8(p[1]))
356 356
 	case mstr16:
357
-		p, err = src.r.Next(3)
357
+		p, err = src.R.Next(3)
358 358
 		if err != nil {
359 359
 			return
360 360
 		}
361 361
 		read = int(big.Uint16(p[1:]))
362 362
 	case mstr32:
363
-		p, err = src.r.Next(5)
363
+		p, err = src.R.Next(5)
364 364
 		if err != nil {
365 365
 			return
366 366
 		}
... ...
@@ -370,7 +370,7 @@ func rwString(dst jsWriter, src *Reader) (n int, err error) {
370 370
 		return
371 371
 	}
372 372
 write:
373
-	p, err = src.r.Next(read)
373
+	p, err = src.R.Next(read)
374 374
 	if err != nil {
375 375
 		return
376 376
 	}
... ...
@@ -1,11 +1,105 @@
1 1
 package msgp
2 2
 
3 3
 import (
4
+	"math"
4 5
 	"strconv"
5 6
 )
6 7
 
7 8
 // The portable parts of the Number implementation
8 9
 
10
+// Number can be
11
+// an int64, uint64, float32,
12
+// or float64 internally.
13
+// It can decode itself
14
+// from any of the native
15
+// messagepack number types.
16
+// The zero-value of Number
17
+// is Int(0). Using the equality
18
+// operator with Number compares
19
+// both the type and the value
20
+// of the number.
21
+type Number struct {
22
+	// internally, this
23
+	// is just a tagged union.
24
+	// the raw bits of the number
25
+	// are stored the same way regardless.
26
+	bits uint64
27
+	typ  Type
28
+}
29
+
30
+// AsInt sets the number to an int64.
31
+func (n *Number) AsInt(i int64) {
32
+
33
+	// we always store int(0)
34
+	// as {0, InvalidType} in
35
+	// order to preserve
36
+	// the behavior of the == operator
37
+	if i == 0 {
38
+		n.typ = InvalidType
39
+		n.bits = 0
40
+		return
41
+	}
42
+
43
+	n.typ = IntType
44
+	n.bits = uint64(i)
45
+}
46
+
47
+// AsUint sets the number to a uint64.
48
+func (n *Number) AsUint(u uint64) {
49
+	n.typ = UintType
50
+	n.bits = u
51
+}
52
+
53
+// AsFloat32 sets the value of the number
54
+// to a float32.
55
+func (n *Number) AsFloat32(f float32) {
56
+	n.typ = Float32Type
57
+	n.bits = uint64(math.Float32bits(f))
58
+}
59
+
60
+// AsFloat64 sets the value of the
61
+// number to a float64.
62
+func (n *Number) AsFloat64(f float64) {
63
+	n.typ = Float64Type
64
+	n.bits = math.Float64bits(f)
65
+}
66
+
67
+// Int casts the number as an int64, and
68
+// returns whether or not that was the
69
+// underlying type.
70
+func (n *Number) Int() (int64, bool) {
71
+	return int64(n.bits), n.typ == IntType || n.typ == InvalidType
72
+}
73
+
74
+// Uint casts the number as a uint64, and returns
75
+// whether or not that was the underlying type.
76
+func (n *Number) Uint() (uint64, bool) {
77
+	return n.bits, n.typ == UintType
78
+}
79
+
80
+// Float casts the number to a float64, and
81
+// returns whether or not that was the underlying
82
+// type (either a float64 or a float32).
83
+func (n *Number) Float() (float64, bool) {
84
+	switch n.typ {
85
+	case Float32Type:
86
+		return float64(math.Float32frombits(uint32(n.bits))), true
87
+	case Float64Type:
88
+		return math.Float64frombits(n.bits), true
89
+	default:
90
+		return 0.0, false
91
+	}
92
+}
93
+
94
+// Type will return one of:
95
+// Float64Type, Float32Type, UintType, or IntType.
96
+func (n *Number) Type() Type {
97
+	if n.typ == InvalidType {
98
+		return IntType
99
+	}
100
+	return n.typ
101
+}
102
+
9 103
 // DecodeMsg implements msgp.Decodable
10 104
 func (n *Number) DecodeMsg(r *Reader) error {
11 105
 	typ, err := r.NextType()
... ...
@@ -83,6 +177,38 @@ func (n *Number) UnmarshalMsg(b []byte) ([]byte, error) {
83 83
 	}
84 84
 }
85 85
 
86
+// MarshalMsg implements msgp.Marshaler
87
+func (n *Number) MarshalMsg(b []byte) ([]byte, error) {
88
+	switch n.typ {
89
+	case IntType:
90
+		return AppendInt64(b, int64(n.bits)), nil
91
+	case UintType:
92
+		return AppendUint64(b, uint64(n.bits)), nil
93
+	case Float64Type:
94
+		return AppendFloat64(b, math.Float64frombits(n.bits)), nil
95
+	case Float32Type:
96
+		return AppendFloat32(b, math.Float32frombits(uint32(n.bits))), nil
97
+	default:
98
+		return AppendInt64(b, 0), nil
99
+	}
100
+}
101
+
102
+// EncodeMsg implements msgp.Encodable
103
+func (n *Number) EncodeMsg(w *Writer) error {
104
+	switch n.typ {
105
+	case IntType:
106
+		return w.WriteInt64(int64(n.bits))
107
+	case UintType:
108
+		return w.WriteUint64(n.bits)
109
+	case Float64Type:
110
+		return w.WriteFloat64(math.Float64frombits(n.bits))
111
+	case Float32Type:
112
+		return w.WriteFloat32(math.Float32frombits(uint32(n.bits)))
113
+	default:
114
+		return w.WriteInt64(0)
115
+	}
116
+}
117
+
86 118
 // Msgsize implements msgp.Sizer
87 119
 func (n *Number) Msgsize() int {
88 120
 	switch n.typ {
... ...
@@ -121,6 +247,7 @@ func (n *Number) MarshalJSON() ([]byte, error) {
121 121
 	}
122 122
 }
123 123
 
124
+// String implements fmt.Stringer
124 125
 func (n *Number) String() string {
125 126
 	switch n.typ {
126 127
 	case InvalidType:
127 128
deleted file mode 100644
... ...
@@ -1,101 +0,0 @@
1
-// +build appengine
2
-
3
-package msgp
4
-
5
-// let's just assume appengine
6
-// uses 64-bit hardware...
7
-const smallint = false
8
-
9
-func UnsafeString(b []byte) string {
10
-	return string(b)
11
-}
12
-
13
-func UnsafeBytes(s string) []byte {
14
-	return []byte(s)
15
-}
16
-
17
-type Number struct {
18
-	ibits uint64  // zero or bits
19
-	fbits float64 // zero or bits
20
-	typ   Type    // zero or type
21
-}
22
-
23
-func (n *Number) AsFloat64(f float64) {
24
-	n.typ = Float64Type
25
-	n.fbits = f
26
-	n.ibits = 0
27
-}
28
-
29
-func (n *Number) AsFloat32(f float32) {
30
-	n.typ = Float32Type
31
-	n.fbits = float64(f)
32
-	n.ibits = 0
33
-}
34
-
35
-func (n *Number) AsInt(i int64) {
36
-	n.fbits = 0
37
-	if i == 0 {
38
-		n.typ = InvalidType
39
-		n.ibits = 0
40
-		return
41
-	}
42
-	n.ibits = uint64(i)
43
-	n.typ = IntType
44
-}
45
-
46
-func (n *Number) AsUint(u uint64) {
47
-	n.ibits = u
48
-	n.fbits = 0
49
-	n.typ = UintType
50
-}
51
-
52
-func (n *Number) Float() (float64, bool) {
53
-	return n.fbits, n.typ == Float64Type || n.typ == Float32Type
54
-}
55
-
56
-func (n *Number) Int() (int64, bool) {
57
-	return int64(n.ibits), n.typ == IntType
58
-}
59
-
60
-func (n *Number) Uint() (uint64, bool) {
61
-	return n.ibits, n.typ == UintType
62
-}
63
-
64
-func (n *Number) Type() Type {
65
-	if n.typ == InvalidType {
66
-		return IntType
67
-	}
68
-	return n.typ
69
-}
70
-
71
-func (n *Number) MarshalMsg(o []byte) ([]byte, error) {
72
-	switch n.typ {
73
-	case InvalidType:
74
-		return AppendInt64(o, 0), nil
75
-	case IntType:
76
-		return AppendInt64(o, int64(n.ibits)), nil
77
-	case UintType:
78
-		return AppendUint64(o, n.ibits), nil
79
-	case Float32Type:
80
-		return AppendFloat32(o, float32(n.fbits)), nil
81
-	case Float64Type:
82
-		return AppendFloat64(o, n.fbits), nil
83
-	}
84
-	panic("unreachable code!")
85
-}
86
-
87
-func (n *Number) EncodeMsg(w *Writer) error {
88
-	switch n.typ {
89
-	case InvalidType:
90
-		return w.WriteInt64(0)
91
-	case IntType:
92
-		return w.WriteInt64(int64(n.ibits))
93
-	case UintType:
94
-		return w.WriteUint64(n.ibits)
95
-	case Float32Type:
96
-		return w.WriteFloat32(float32(n.fbits))
97
-	case Float64Type:
98
-		return w.WriteFloat64(n.fbits)
99
-	}
100
-	panic("unreachable code!")
101
-}
102 1
deleted file mode 100644
... ...
@@ -1,159 +0,0 @@
1
-// +build !appengine
2
-
3
-package msgp
4
-
5
-import (
6
-	"reflect"
7
-	"unsafe"
8
-)
9
-
10
-const (
11
-	// spec says int and uint are always
12
-	// the same size, but that int/uint
13
-	// size may not be machine word size
14
-	smallint = unsafe.Sizeof(int(0)) == 4
15
-)
16
-
17
-// UnsafeString returns the byte slice as a volatile string
18
-// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
19
-// THIS IS EVIL CODE.
20
-// YOU HAVE BEEN WARNED.
21
-func UnsafeString(b []byte) string {
22
-	return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: uintptr(unsafe.Pointer(&b[0])), Len: len(b)}))
23
-}
24
-
25
-// UnsafeBytes returns the string as a byte slice
26
-// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
27
-// THIS IS EVIL CODE.
28
-// YOU HAVE BEEN WARNED.
29
-func UnsafeBytes(s string) []byte {
30
-	return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
31
-		Len:  len(s),
32
-		Cap:  len(s),
33
-		Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data,
34
-	}))
35
-}
36
-
37
-// Number can be
38
-// an int64, uint64, float32,
39
-// or float64 internally.
40
-// It can decode itself
41
-// from any of the native
42
-// messagepack number types.
43
-// The zero-value of Number
44
-// is Int(0). Using the equality
45
-// operator with Number compares
46
-// both the type and the value
47
-// of the number.
48
-type Number struct {
49
-	// internally, this
50
-	// is just a tagged union.
51
-	// the raw bits of the number
52
-	// are stored the same way regardless.
53
-	bits uint64
54
-	typ  Type
55
-}
56
-
57
-// AsFloat64 sets the number to
58
-// a float64.
59
-func (n *Number) AsFloat64(f float64) {
60
-	n.typ = Float64Type
61
-	n.bits = *(*uint64)(unsafe.Pointer(&f))
62
-}
63
-
64
-// AsInt sets the number to an int64.
65
-func (n *Number) AsInt(i int64) {
66
-
67
-	// we always store int(0)
68
-	// as {0, InvalidType} in
69
-	// order to preserve
70
-	// the behavior of the == operator
71
-	if i == 0 {
72
-		n.typ = InvalidType
73
-		n.bits = 0
74
-		return
75
-	}
76
-
77
-	n.typ = IntType
78
-	n.bits = uint64(i)
79
-}
80
-
81
-// AsUint sets the number to a uint64.
82
-func (n *Number) AsUint(u uint64) {
83
-	n.typ = UintType
84
-	n.bits = u
85
-}
86
-
87
-// AsFloat32 sets the number to a float32.
88
-func (n *Number) AsFloat32(f float32) {
89
-	n.typ = Float32Type
90
-	g := float64(f)
91
-	n.bits = *(*uint64)(unsafe.Pointer(&g))
92
-}
93
-
94
-// Type will return one of:
95
-// Float64Type, Float32Type, UintType, or IntType.
96
-func (n *Number) Type() Type {
97
-	if n.typ == InvalidType {
98
-		return IntType
99
-	}
100
-	return n.typ
101
-}
102
-
103
-// Float casts the number of the float,
104
-// and returns whether or not that was
105
-// the underlying type. (This is legal
106
-// for both float32 and float64 types.)
107
-func (n *Number) Float() (float64, bool) {
108
-	return *(*float64)(unsafe.Pointer(&n.bits)), n.typ == Float64Type || n.typ == Float32Type
109
-}
110
-
111
-// Int casts the number as an int64, and
112
-// returns whether or not that was the
113
-// underlying type.
114
-func (n *Number) Int() (int64, bool) {
115
-	return int64(n.bits), n.typ == IntType || n.typ == InvalidType
116
-}
117
-
118
-// Uint casts the number as a uint64, and returns
119
-// whether or not that was the underlying type.
120
-func (n *Number) Uint() (uint64, bool) {
121
-	return n.bits, n.typ == UintType
122
-}
123
-
124
-// EncodeMsg implements msgp.Encodable
125
-func (n *Number) EncodeMsg(w *Writer) error {
126
-	switch n.typ {
127
-	case InvalidType:
128
-		return w.WriteInt(0)
129
-	case IntType:
130
-		return w.WriteInt64(int64(n.bits))
131
-	case UintType:
132
-		return w.WriteUint64(n.bits)
133
-	case Float64Type:
134
-		return w.WriteFloat64(*(*float64)(unsafe.Pointer(&n.bits)))
135
-	case Float32Type:
136
-		return w.WriteFloat32(float32(*(*float64)(unsafe.Pointer(&n.bits))))
137
-	default:
138
-		// this should never ever happen
139
-		panic("(*Number).typ is invalid")
140
-	}
141
-}
142
-
143
-// MarshalMsg implements msgp.Marshaler
144
-func (n *Number) MarshalMsg(b []byte) ([]byte, error) {
145
-	switch n.typ {
146
-	case InvalidType:
147
-		return AppendInt(b, 0), nil
148
-	case IntType:
149
-		return AppendInt64(b, int64(n.bits)), nil
150
-	case UintType:
151
-		return AppendUint64(b, n.bits), nil
152
-	case Float64Type:
153
-		return AppendFloat64(b, *(*float64)(unsafe.Pointer(&n.bits))), nil
154
-	case Float32Type:
155
-		return AppendFloat32(b, float32(*(*float64)(unsafe.Pointer(&n.bits)))), nil
156
-	default:
157
-		panic("(*Number).typ is invalid")
158
-	}
159
-}
... ...
@@ -1,11 +1,12 @@
1 1
 package msgp
2 2
 
3 3
 import (
4
-	"github.com/philhofer/fwd"
5 4
 	"io"
6 5
 	"math"
7 6
 	"sync"
8 7
 	"time"
8
+
9
+	"github.com/philhofer/fwd"
9 10
 )
10 11
 
11 12
 // where we keep old *Readers
... ...
@@ -111,10 +112,10 @@ func Decode(r io.Reader, d Decodable) error {
111 111
 // reader will be buffered.
112 112
 func NewReader(r io.Reader) *Reader {
113 113
 	p := readerPool.Get().(*Reader)
114
-	if p.r == nil {
115
-		p.r = fwd.NewReader(r)
114
+	if p.R == nil {
115
+		p.R = fwd.NewReader(r)
116 116
 	} else {
117
-		p.r.Reset(r)
117
+		p.R.Reset(r)
118 118
 	}
119 119
 	return p
120 120
 }
... ...
@@ -122,39 +123,96 @@ func NewReader(r io.Reader) *Reader {
122 122
 // NewReaderSize returns a *Reader with a buffer of the given size.
123 123
 // (This is vastly preferable to passing the decoder a reader that is already buffered.)
124 124
 func NewReaderSize(r io.Reader, sz int) *Reader {
125
-	return &Reader{r: fwd.NewReaderSize(r, sz)}
125
+	return &Reader{R: fwd.NewReaderSize(r, sz)}
126 126
 }
127 127
 
128 128
 // Reader wraps an io.Reader and provides
129 129
 // methods to read MessagePack-encoded values
130 130
 // from it. Readers are buffered.
131 131
 type Reader struct {
132
-	r       *fwd.Reader
132
+	// R is the buffered reader
133
+	// that the Reader uses
134
+	// to decode MessagePack.
135
+	// The Reader itself
136
+	// is stateless; all the
137
+	// buffering is done
138
+	// within R.
139
+	R       *fwd.Reader
133 140
 	scratch []byte
134 141
 }
135 142
 
136 143
 // Read implements `io.Reader`
137 144
 func (m *Reader) Read(p []byte) (int, error) {
138
-	return m.r.Read(p)
145
+	return m.R.Read(p)
146
+}
147
+
148
+// CopyNext reads the next object from m without decoding it and writes it to w.
149
+// It avoids unnecessary copies internally.
150
+func (m *Reader) CopyNext(w io.Writer) (int64, error) {
151
+	sz, o, err := getNextSize(m.R)
152
+	if err != nil {
153
+		return 0, err
154
+	}
155
+
156
+	var n int64
157
+	// Opportunistic optimization: if we can fit the whole thing in the m.R
158
+	// buffer, then just get a pointer to that, and pass it to w.Write,
159
+	// avoiding an allocation.
160
+	if int(sz) <= m.R.BufferSize() {
161
+		var nn int
162
+		var buf []byte
163
+		buf, err = m.R.Next(int(sz))
164
+		if err != nil {
165
+			if err == io.ErrUnexpectedEOF {
166
+				err = ErrShortBytes
167
+			}
168
+			return 0, err
169
+		}
170
+		nn, err = w.Write(buf)
171
+		n += int64(nn)
172
+	} else {
173
+		// Fall back to io.CopyN.
174
+		// May avoid allocating if w is a ReaderFrom (e.g. bytes.Buffer)
175
+		n, err = io.CopyN(w, m.R, int64(sz))
176
+		if err == io.ErrUnexpectedEOF {
177
+			err = ErrShortBytes
178
+		}
179
+	}
180
+	if err != nil {
181
+		return n, err
182
+	} else if n < int64(sz) {
183
+		return n, io.ErrShortWrite
184
+	}
185
+
186
+	// for maps and slices, read elements
187
+	for x := uintptr(0); x < o; x++ {
188
+		var n2 int64
189
+		n2, err = m.CopyNext(w)
190
+		if err != nil {
191
+			return n, err
192
+		}
193
+		n += n2
194
+	}
195
+	return n, nil
139 196
 }
140 197
 
141 198
 // ReadFull implements `io.ReadFull`
142 199
 func (m *Reader) ReadFull(p []byte) (int, error) {
143
-	return m.r.ReadFull(p)
200
+	return m.R.ReadFull(p)
144 201
 }
145 202
 
146 203
 // Reset resets the underlying reader.
147
-func (m *Reader) Reset(r io.Reader) { m.r.Reset(r) }
204
+func (m *Reader) Reset(r io.Reader) { m.R.Reset(r) }
148 205
 
149 206
 // Buffered returns the number of bytes currently in the read buffer.
150
-func (m *Reader) Buffered() int { return m.r.Buffered() }
207
+func (m *Reader) Buffered() int { return m.R.Buffered() }
151 208
 
152 209
 // BufferSize returns the capacity of the read buffer.
153
-func (m *Reader) BufferSize() int { return m.r.BufferSize() }
210
+func (m *Reader) BufferSize() int { return m.R.BufferSize() }
154 211
 
155 212
 // NextType returns the next object type to be decoded.
156 213
 func (m *Reader) NextType() (Type, error) {
157
-	p, err := m.r.Peek(1)
214
+	p, err := m.R.Peek(1)
158 215
 	if err != nil {
159 216
 		return InvalidType, err
160 217
 	}
... ...
@@ -182,12 +240,14 @@ func (m *Reader) NextType() (Type, error) {
182 182
 // IsNil returns whether or not
183 183
 // the next byte is a null messagepack byte
184 184
 func (m *Reader) IsNil() bool {
185
-	p, err := m.r.Peek(1)
185
+	p, err := m.R.Peek(1)
186 186
 	return err == nil && p[0] == mnil
187 187
 }
188 188
 
189
+// getNextSize returns the size of the next object on the wire.
189 190
 // returns (obj size, obj elements, error)
190 191
 // only maps and arrays have non-zero obj elements
192
+// for maps and arrays, obj size does not include elements
191 193
 //
192 194
 // use uintptr b/c it's guaranteed to be large enough
193 195
 // to hold whatever we can fit in memory.
... ...
@@ -243,8 +303,8 @@ func (m *Reader) Skip() error {
243 243
 	// we can use the faster
244 244
 	// method if we have enough
245 245
 	// buffered data
246
-	if m.r.Buffered() >= 5 {
247
-		p, err = m.r.Peek(5)
246
+	if m.R.Buffered() >= 5 {
247
+		p, err = m.R.Peek(5)
248 248
 		if err != nil {
249 249
 			return err
250 250
 		}
... ...
@@ -253,7 +313,7 @@ func (m *Reader) Skip() error {
253 253
 			return err
254 254
 		}
255 255
 	} else {
256
-		v, o, err = getNextSize(m.r)
256
+		v, o, err = getNextSize(m.R)
257 257
 		if err != nil {
258 258
 			return err
259 259
 		}
... ...
@@ -261,7 +321,7 @@ func (m *Reader) Skip() error {
261 261
 
262 262
 	// 'v' is always non-zero
263 263
 	// if err == nil
264
-	_, err = m.r.Skip(int(v))
264
+	_, err = m.R.Skip(int(v))
265 265
 	if err != nil {
266 266
 		return err
267 267
 	}
... ...
@@ -284,26 +344,26 @@ func (m *Reader) Skip() error {
284 284
 func (m *Reader) ReadMapHeader() (sz uint32, err error) {
285 285
 	var p []byte
286 286
 	var lead byte
287
-	p, err = m.r.Peek(1)
287
+	p, err = m.R.Peek(1)
288 288
 	if err != nil {
289 289
 		return
290 290
 	}
291 291
 	lead = p[0]
292 292
 	if isfixmap(lead) {
293 293
 		sz = uint32(rfixmap(lead))
294
-		_, err = m.r.Skip(1)
294
+		_, err = m.R.Skip(1)
295 295
 		return
296 296
 	}
297 297
 	switch lead {
298 298
 	case mmap16:
299
-		p, err = m.r.Next(3)
299
+		p, err = m.R.Next(3)
300 300
 		if err != nil {
301 301
 			return
302 302
 		}
303 303
 		sz = uint32(big.Uint16(p[1:]))
304 304
 		return
305 305
 	case mmap32:
306
-		p, err = m.r.Next(5)
306
+		p, err = m.R.Next(5)
307 307
 		if err != nil {
308 308
 			return
309 309
 		}
... ...
@@ -338,7 +398,7 @@ func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) {
338 338
 // method; writing into the returned slice may
339 339
 // corrupt future reads.
340 340
 func (m *Reader) ReadMapKeyPtr() ([]byte, error) {
341
-	p, err := m.r.Peek(1)
341
+	p, err := m.R.Peek(1)
342 342
 	if err != nil {
343 343
 		return nil, err
344 344
 	}
... ...
@@ -346,24 +406,24 @@ func (m *Reader) ReadMapKeyPtr() ([]byte, error) {
346 346
 	var read int
347 347
 	if isfixstr(lead) {
348 348
 		read = int(rfixstr(lead))
349
-		m.r.Skip(1)
349
+		m.R.Skip(1)
350 350
 		goto fill
351 351
 	}
352 352
 	switch lead {
353 353
 	case mstr8, mbin8:
354
-		p, err = m.r.Next(2)
354
+		p, err = m.R.Next(2)
355 355
 		if err != nil {
356 356
 			return nil, err
357 357
 		}
358 358
 		read = int(p[1])
359 359
 	case mstr16, mbin16:
360
-		p, err = m.r.Next(3)
360
+		p, err = m.R.Next(3)
361 361
 		if err != nil {
362 362
 			return nil, err
363 363
 		}
364 364
 		read = int(big.Uint16(p[1:]))
365 365
 	case mstr32, mbin32:
366
-		p, err = m.r.Next(5)
366
+		p, err = m.R.Next(5)
367 367
 		if err != nil {
368 368
 			return nil, err
369 369
 		}
... ...
@@ -375,7 +435,7 @@ fill:
375 375
 	if read == 0 {
376 376
 		return nil, ErrShortBytes
377 377
 	}
378
-	return m.r.Next(read)
378
+	return m.R.Next(read)
379 379
 }
380 380
 
381 381
 // ReadArrayHeader reads the next object as an
... ...
@@ -384,19 +444,19 @@ fill:
384 384
 func (m *Reader) ReadArrayHeader() (sz uint32, err error) {
385 385
 	var lead byte
386 386
 	var p []byte
387
-	p, err = m.r.Peek(1)
387
+	p, err = m.R.Peek(1)
388 388
 	if err != nil {
389 389
 		return
390 390
 	}
391 391
 	lead = p[0]
392 392
 	if isfixarray(lead) {
393 393
 		sz = uint32(rfixarray(lead))
394
-		_, err = m.r.Skip(1)
394
+		_, err = m.R.Skip(1)
395 395
 		return
396 396
 	}
397 397
 	switch lead {
398 398
 	case marray16:
399
-		p, err = m.r.Next(3)
399
+		p, err = m.R.Next(3)
400 400
 		if err != nil {
401 401
 			return
402 402
 		}
... ...
@@ -404,7 +464,7 @@ func (m *Reader) ReadArrayHeader() (sz uint32, err error) {
404 404
 		return
405 405
 
406 406
 	case marray32:
407
-		p, err = m.r.Next(5)
407
+		p, err = m.R.Next(5)
408 408
 		if err != nil {
409 409
 			return
410 410
 		}
... ...
@@ -419,14 +479,14 @@ func (m *Reader) ReadArrayHeader() (sz uint32, err error) {
419 419
 
420 420
 // ReadNil reads a 'nil' MessagePack byte from the reader
421 421
 func (m *Reader) ReadNil() error {
422
-	p, err := m.r.Peek(1)
422
+	p, err := m.R.Peek(1)
423 423
 	if err != nil {
424 424
 		return err
425 425
 	}
426 426
 	if p[0] != mnil {
427 427
 		return badPrefix(NilType, p[0])
428 428
 	}
429
-	_, err = m.r.Skip(1)
429
+	_, err = m.R.Skip(1)
430 430
 	return err
431 431
 }
432 432
 
... ...
@@ -435,7 +495,7 @@ func (m *Reader) ReadNil() error {
435 435
 // it will be up-cast to a float64.)
436 436
 func (m *Reader) ReadFloat64() (f float64, err error) {
437 437
 	var p []byte
438
-	p, err = m.r.Peek(9)
438
+	p, err = m.R.Peek(9)
439 439
 	if err != nil {
440 440
 		// we'll allow a coversion from float32 to float64,
441 441
 		// since we don't lose any precision
... ...
@@ -455,14 +515,14 @@ func (m *Reader) ReadFloat64() (f float64, err error) {
455 455
 		return
456 456
 	}
457 457
 	f = math.Float64frombits(getMuint64(p))
458
-	_, err = m.r.Skip(9)
458
+	_, err = m.R.Skip(9)
459 459
 	return
460 460
 }
461 461
 
462 462
 // ReadFloat32 reads a float32 from the reader
463 463
 func (m *Reader) ReadFloat32() (f float32, err error) {
464 464
 	var p []byte
465
-	p, err = m.r.Peek(5)
465
+	p, err = m.R.Peek(5)
466 466
 	if err != nil {
467 467
 		return
468 468
 	}
... ...
@@ -471,14 +531,14 @@ func (m *Reader) ReadFloat32() (f float32, err error) {
471 471
 		return
472 472
 	}
473 473
 	f = math.Float32frombits(getMuint32(p))
474
-	_, err = m.r.Skip(5)
474
+	_, err = m.R.Skip(5)
475 475
 	return
476 476
 }
477 477
 
478 478
 // ReadBool reads a bool from the reader
479 479
 func (m *Reader) ReadBool() (b bool, err error) {
480 480
 	var p []byte
481
-	p, err = m.r.Peek(1)
481
+	p, err = m.R.Peek(1)
482 482
 	if err != nil {
483 483
 		return
484 484
 	}
... ...
@@ -490,7 +550,7 @@ func (m *Reader) ReadBool() (b bool, err error) {
490 490
 		err = badPrefix(BoolType, p[0])
491 491
 		return
492 492
 	}
493
-	_, err = m.r.Skip(1)
493
+	_, err = m.R.Skip(1)
494 494
 	return
495 495
 }
496 496
 
... ...
@@ -498,7 +558,7 @@ func (m *Reader) ReadBool() (b bool, err error) {
498 498
 func (m *Reader) ReadInt64() (i int64, err error) {
499 499
 	var p []byte
500 500
 	var lead byte
501
-	p, err = m.r.Peek(1)
501
+	p, err = m.R.Peek(1)
502 502
 	if err != nil {
503 503
 		return
504 504
 	}
... ...
@@ -506,17 +566,17 @@ func (m *Reader) ReadInt64() (i int64, err error) {
506 506
 
507 507
 	if isfixint(lead) {
508 508
 		i = int64(rfixint(lead))
509
-		_, err = m.r.Skip(1)
509
+		_, err = m.R.Skip(1)
510 510
 		return
511 511
 	} else if isnfixint(lead) {
512 512
 		i = int64(rnfixint(lead))
513
-		_, err = m.r.Skip(1)
513
+		_, err = m.R.Skip(1)
514 514
 		return
515 515
 	}
516 516
 
517 517
 	switch lead {
518 518
 	case mint8:
519
-		p, err = m.r.Next(2)
519
+		p, err = m.R.Next(2)
520 520
 		if err != nil {
521 521
 			return
522 522
 		}
... ...
@@ -524,7 +584,7 @@ func (m *Reader) ReadInt64() (i int64, err error) {
524 524
 		return
525 525
 
526 526
 	case mint16:
527
-		p, err = m.r.Next(3)
527
+		p, err = m.R.Next(3)
528 528
 		if err != nil {
529 529
 			return
530 530
 		}
... ...
@@ -532,7 +592,7 @@ func (m *Reader) ReadInt64() (i int64, err error) {
532 532
 		return
533 533
 
534 534
 	case mint32:
535
-		p, err = m.r.Next(5)
535
+		p, err = m.R.Next(5)
536 536
 		if err != nil {
537 537
 			return
538 538
 		}
... ...
@@ -540,7 +600,7 @@ func (m *Reader) ReadInt64() (i int64, err error) {
540 540
 		return
541 541
 
542 542
 	case mint64:
543
-		p, err = m.r.Next(9)
543
+		p, err = m.R.Next(9)
544 544
 		if err != nil {
545 545
 			return
546 546
 		}
... ...
@@ -607,19 +667,19 @@ func (m *Reader) ReadInt() (i int, err error) {
607 607
 func (m *Reader) ReadUint64() (u uint64, err error) {
608 608
 	var p []byte
609 609
 	var lead byte
610
-	p, err = m.r.Peek(1)
610
+	p, err = m.R.Peek(1)
611 611
 	if err != nil {
612 612
 		return
613 613
 	}
614 614
 	lead = p[0]
615 615
 	if isfixint(lead) {
616 616
 		u = uint64(rfixint(lead))
617
-		_, err = m.r.Skip(1)
617
+		_, err = m.R.Skip(1)
618 618
 		return
619 619
 	}
620 620
 	switch lead {
621 621
 	case muint8:
622
-		p, err = m.r.Next(2)
622
+		p, err = m.R.Next(2)
623 623
 		if err != nil {
624 624
 			return
625 625
 		}
... ...
@@ -627,7 +687,7 @@ func (m *Reader) ReadUint64() (u uint64, err error) {
627 627
 		return
628 628
 
629 629
 	case muint16:
630
-		p, err = m.r.Next(3)
630
+		p, err = m.R.Next(3)
631 631
 		if err != nil {
632 632
 			return
633 633
 		}
... ...
@@ -635,7 +695,7 @@ func (m *Reader) ReadUint64() (u uint64, err error) {
635 635
 		return
636 636
 
637 637
 	case muint32:
638
-		p, err = m.r.Next(5)
638
+		p, err = m.R.Next(5)
639 639
 		if err != nil {
640 640
 			return
641 641
 		}
... ...
@@ -643,7 +703,7 @@ func (m *Reader) ReadUint64() (u uint64, err error) {
643 643
 		return
644 644
 
645 645
 	case muint64:
646
-		p, err = m.r.Next(9)
646
+		p, err = m.R.Next(9)
647 647
 		if err != nil {
648 648
 			return
649 649
 		}
... ...
@@ -707,6 +767,10 @@ func (m *Reader) ReadUint() (u uint, err error) {
707 707
 	return
708 708
 }
709 709
 
710
+// ReadByte is analogous to ReadUint8.
711
+//
712
+// NOTE: this is *not* an implementation
713
+// of io.ByteReader.
710 714
 func (m *Reader) ReadByte() (b byte, err error) {
711 715
 	var in uint64
712 716
 	in, err = m.ReadUint64()
... ...
@@ -724,7 +788,7 @@ func (m *Reader) ReadByte() (b byte, err error) {
724 724
 func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) {
725 725
 	var p []byte
726 726
 	var lead byte
727
-	p, err = m.r.Peek(2)
727
+	p, err = m.R.Peek(2)
728 728
 	if err != nil {
729 729
 		return
730 730
 	}
... ...
@@ -733,15 +797,15 @@ func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) {
733 733
 	switch lead {
734 734
 	case mbin8:
735 735
 		read = int64(p[1])
736
-		m.r.Skip(2)
736
+		m.R.Skip(2)
737 737
 	case mbin16:
738
-		p, err = m.r.Next(3)
738
+		p, err = m.R.Next(3)
739 739
 		if err != nil {
740 740
 			return
741 741
 		}
742 742
 		read = int64(big.Uint16(p[1:]))
743 743
 	case mbin32:
744
-		p, err = m.r.Next(5)
744
+		p, err = m.R.Next(5)
745 745
 		if err != nil {
746 746
 			return
747 747
 		}
... ...
@@ -755,16 +819,55 @@ func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) {
755 755
 	} else {
756 756
 		b = scratch[0:read]
757 757
 	}
758
-	_, err = m.r.ReadFull(b)
758
+	_, err = m.R.ReadFull(b)
759 759
 	return
760 760
 }
761 761
 
762
+// ReadBytesHeader reads the size header
763
+// of a MessagePack 'bin' object. The user
764
+// is responsible for dealing with the next
765
+// 'sz' bytes from the reader in an application-specific
766
+// way.
767
+func (m *Reader) ReadBytesHeader() (sz uint32, err error) {
768
+	var p []byte
769
+	p, err = m.R.Peek(1)
770
+	if err != nil {
771
+		return
772
+	}
773
+	switch p[0] {
774
+	case mbin8:
775
+		p, err = m.R.Next(2)
776
+		if err != nil {
777
+			return
778
+		}
779
+		sz = uint32(p[1])
780
+		return
781
+	case mbin16:
782
+		p, err = m.R.Next(3)
783
+		if err != nil {
784
+			return
785
+		}
786
+		sz = uint32(big.Uint16(p[1:]))
787
+		return
788
+	case mbin32:
789
+		p, err = m.R.Next(5)
790
+		if err != nil {
791
+			return
792
+		}
793
+		sz = uint32(big.Uint32(p[1:]))
794
+		return
795
+	default:
796
+		err = badPrefix(BinType, p[0])
797
+		return
798
+	}
799
+}
800
+
762 801
 // ReadExactBytes reads a MessagePack 'bin'-encoded
763 802
 // object off of the wire into the provided slice. An
764 803
 // ArrayError will be returned if the object is not
765 804
 // exactly the length of the input slice.
766 805
 func (m *Reader) ReadExactBytes(into []byte) error {
767
-	p, err := m.r.Peek(2)
806
+	p, err := m.R.Peek(2)
768 807
 	if err != nil {
769 808
 		return err
770 809
 	}
... ...
@@ -776,14 +879,14 @@ func (m *Reader) ReadExactBytes(into []byte) error {
776 776
 		read = int64(p[1])
777 777
 		skip = 2
778 778
 	case mbin16:
779
-		p, err = m.r.Peek(3)
779
+		p, err = m.R.Peek(3)
780 780
 		if err != nil {
781 781
 			return err
782 782
 		}
783 783
 		read = int64(big.Uint16(p[1:]))
784 784
 		skip = 3
785 785
 	case mbin32:
786
-		p, err = m.r.Peek(5)
786
+		p, err = m.R.Peek(5)
787 787
 		if err != nil {
788 788
 			return err
789 789
 		}
... ...
@@ -795,8 +898,8 @@ func (m *Reader) ReadExactBytes(into []byte) error {
795 795
 	if read != int64(len(into)) {
796 796
 		return ArrayError{Wanted: uint32(len(into)), Got: uint32(read)}
797 797
 	}
798
-	m.r.Skip(skip)
799
-	_, err = m.r.ReadFull(into)
798
+	m.R.Skip(skip)
799
+	_, err = m.R.ReadFull(into)
800 800
 	return err
801 801
 }
802 802
 
... ...
@@ -806,7 +909,7 @@ func (m *Reader) ReadExactBytes(into []byte) error {
806 806
 func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) {
807 807
 	var p []byte
808 808
 	var lead byte
809
-	p, err = m.r.Peek(1)
809
+	p, err = m.R.Peek(1)
810 810
 	if err != nil {
811 811
 		return
812 812
 	}
... ...
@@ -815,25 +918,25 @@ func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) {
815 815
 
816 816
 	if isfixstr(lead) {
817 817
 		read = int64(rfixstr(lead))
818
-		m.r.Skip(1)
818
+		m.R.Skip(1)
819 819
 		goto fill
820 820
 	}
821 821
 
822 822
 	switch lead {
823 823
 	case mstr8:
824
-		p, err = m.r.Next(2)
824
+		p, err = m.R.Next(2)
825 825
 		if err != nil {
826 826
 			return
827 827
 		}
828 828
 		read = int64(uint8(p[1]))
829 829
 	case mstr16:
830
-		p, err = m.r.Next(3)
830
+		p, err = m.R.Next(3)
831 831
 		if err != nil {
832 832
 			return
833 833
 		}
834 834
 		read = int64(big.Uint16(p[1:]))
835 835
 	case mstr32:
836
-		p, err = m.r.Next(5)
836
+		p, err = m.R.Next(5)
837 837
 		if err != nil {
838 838
 			return
839 839
 		}
... ...
@@ -848,16 +951,60 @@ fill:
848 848
 	} else {
849 849
 		b = scratch[0:read]
850 850
 	}
851
-	_, err = m.r.ReadFull(b)
851
+	_, err = m.R.ReadFull(b)
852 852
 	return
853 853
 }
854 854
 
855
+// ReadStringHeader reads a string header
856
+// off of the wire. The user is then responsible
857
+// for dealing with the next 'sz' bytes from
858
+// the reader in an application-specific manner.
859
+func (m *Reader) ReadStringHeader() (sz uint32, err error) {
860
+	var p []byte
861
+	p, err = m.R.Peek(1)
862
+	if err != nil {
863
+		return
864
+	}
865
+	lead := p[0]
866
+	if isfixstr(lead) {
867
+		sz = uint32(rfixstr(lead))
868
+		m.R.Skip(1)
869
+		return
870
+	}
871
+	switch lead {
872
+	case mstr8:
873
+		p, err = m.R.Next(2)
874
+		if err != nil {
875
+			return
876
+		}
877
+		sz = uint32(p[1])
878
+		return
879
+	case mstr16:
880
+		p, err = m.R.Next(3)
881
+		if err != nil {
882
+			return
883
+		}
884
+		sz = uint32(big.Uint16(p[1:]))
885
+		return
886
+	case mstr32:
887
+		p, err = m.R.Next(5)
888
+		if err != nil {
889
+			return
890
+		}
891
+		sz = big.Uint32(p[1:])
892
+		return
893
+	default:
894
+		err = badPrefix(StrType, lead)
895
+		return
896
+	}
897
+}
898
+
855 899
 // ReadString reads a utf-8 string from the reader
856 900
 func (m *Reader) ReadString() (s string, err error) {
857 901
 	var p []byte
858 902
 	var lead byte
859 903
 	var read int64
860
-	p, err = m.r.Peek(1)
904
+	p, err = m.R.Peek(1)
861 905
 	if err != nil {
862 906
 		return
863 907
 	}
... ...
@@ -865,25 +1012,25 @@ func (m *Reader) ReadString() (s string, err error) {
865 865
 
866 866
 	if isfixstr(lead) {
867 867
 		read = int64(rfixstr(lead))
868
-		m.r.Skip(1)
868
+		m.R.Skip(1)
869 869
 		goto fill
870 870
 	}
871 871
 
872 872
 	switch lead {
873 873
 	case mstr8:
874
-		p, err = m.r.Next(2)
874
+		p, err = m.R.Next(2)
875 875
 		if err != nil {
876 876
 			return
877 877
 		}
878 878
 		read = int64(uint8(p[1]))
879 879
 	case mstr16:
880
-		p, err = m.r.Next(3)
880
+		p, err = m.R.Next(3)
881 881
 		if err != nil {
882 882
 			return
883 883
 		}
884 884
 		read = int64(big.Uint16(p[1:]))
885 885
 	case mstr32:
886
-		p, err = m.r.Next(5)
886
+		p, err = m.R.Next(5)
887 887
 		if err != nil {
888 888
 			return
889 889
 		}
... ...
@@ -915,7 +1062,7 @@ fill:
915 915
 	// thus escape analysis *must* conclude that
916 916
 	// 'out' escapes.
917 917
 	out := make([]byte, read)
918
-	_, err = m.r.ReadFull(out)
918
+	_, err = m.R.ReadFull(out)
919 919
 	if err != nil {
920 920
 		return
921 921
 	}
... ...
@@ -926,7 +1073,7 @@ fill:
926 926
 // ReadComplex64 reads a complex64 from the reader
927 927
 func (m *Reader) ReadComplex64() (f complex64, err error) {
928 928
 	var p []byte
929
-	p, err = m.r.Peek(10)
929
+	p, err = m.R.Peek(10)
930 930
 	if err != nil {
931 931
 		return
932 932
 	}
... ...
@@ -940,14 +1087,14 @@ func (m *Reader) ReadComplex64() (f complex64, err error) {
940 940
 	}
941 941
 	f = complex(math.Float32frombits(big.Uint32(p[2:])),
942 942
 		math.Float32frombits(big.Uint32(p[6:])))
943
-	_, err = m.r.Skip(10)
943
+	_, err = m.R.Skip(10)
944 944
 	return
945 945
 }
946 946
 
947 947
 // ReadComplex128 reads a complex128 from the reader
948 948
 func (m *Reader) ReadComplex128() (f complex128, err error) {
949 949
 	var p []byte
950
-	p, err = m.r.Peek(18)
950
+	p, err = m.R.Peek(18)
951 951
 	if err != nil {
952 952
 		return
953 953
 	}
... ...
@@ -961,7 +1108,7 @@ func (m *Reader) ReadComplex128() (f complex128, err error) {
961 961
 	}
962 962
 	f = complex(math.Float64frombits(big.Uint64(p[2:])),
963 963
 		math.Float64frombits(big.Uint64(p[10:])))
964
-	_, err = m.r.Skip(18)
964
+	_, err = m.R.Skip(18)
965 965
 	return
966 966
 }
967 967
 
... ...
@@ -996,7 +1143,7 @@ func (m *Reader) ReadMapStrIntf(mp map[string]interface{}) (err error) {
996 996
 // The returned time's location will be set to time.Local.
997 997
 func (m *Reader) ReadTime() (t time.Time, err error) {
998 998
 	var p []byte
999
-	p, err = m.r.Peek(15)
999
+	p, err = m.R.Peek(15)
1000 1000
 	if err != nil {
1001 1001
 		return
1002 1002
 	}
... ...
@@ -1010,7 +1157,7 @@ func (m *Reader) ReadTime() (t time.Time, err error) {
1010 1010
 	}
1011 1011
 	sec, nsec := getUnix(p[3:])
1012 1012
 	t = time.Unix(sec, int64(nsec)).Local()
1013
-	_, err = m.r.Skip(15)
1013
+	_, err = m.R.Skip(15)
1014 1014
 	return
1015 1015
 }
1016 1016
 
... ...
@@ -117,13 +117,13 @@ func (r Raw) Msgsize() int {
117 117
 }
118 118
 
119 119
 func appendNext(f *Reader, d *[]byte) error {
120
-	amt, o, err := getNextSize(f.r)
120
+	amt, o, err := getNextSize(f.R)
121 121
 	if err != nil {
122 122
 		return err
123 123
 	}
124 124
 	var i int
125 125
 	*d, i = ensure(*d, int(amt))
126
-	_, err = f.r.ReadFull((*d)[i:])
126
+	_, err = f.R.ReadFull((*d)[i:])
127 127
 	if err != nil {
128 128
 		return err
129 129
 	}
... ...
@@ -576,7 +576,7 @@ func ReadUintBytes(b []byte) (uint, []byte, error) {
576 576
 	return uint(u), b, err
577 577
 }
578 578
 
579
-// ReadByteBytes is analagous to ReadUint8Bytes
579
+// ReadByteBytes is analogous to ReadUint8Bytes
580 580
 func ReadByteBytes(b []byte) (byte, []byte, error) {
581 581
 	return ReadUint8Bytes(b)
582 582
 }
... ...
@@ -784,6 +784,22 @@ func ReadStringBytes(b []byte) (string, []byte, error) {
784 784
 	return string(v), o, err
785 785
 }
786 786
 
787
+// ReadStringAsBytes reads a 'str' object
788
+// into a slice of bytes. 'v' is the value of
789
+// the 'str' object, which may reside in memory
790
+// pointed to by 'scratch.' 'o' is the remaining bytes
791
+// in 'b.''
792
+// Possible errors:
793
+// - ErrShortBytes (b not long enough)
794
+// - TypeError{} (not 'str' type)
795
+// - InvalidPrefixError (unknown type marker)
796
+func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) {
797
+	var tmp []byte
798
+	tmp, o, err = ReadStringZC(b)
799
+	v = append(scratch[:0], tmp...)
800
+	return
801
+}
802
+
787 803
 // ReadComplex128Bytes reads a complex128
788 804
 // extension object from 'b' and returns the
789 805
 // remaining bytes.
... ...
@@ -922,14 +938,14 @@ func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) {
922 922
 
923 923
 	case ArrayType:
924 924
 		var sz uint32
925
-		sz, b, err = ReadArrayHeaderBytes(b)
925
+		sz, o, err = ReadArrayHeaderBytes(b)
926 926
 		if err != nil {
927 927
 			return
928 928
 		}
929 929
 		j := make([]interface{}, int(sz))
930 930
 		i = j
931 931
 		for d := range j {
932
-			j[d], b, err = ReadIntfBytes(b)
932
+			j[d], o, err = ReadIntfBytes(o)
933 933
 			if err != nil {
934 934
 				return
935 935
 			}
936 936
new file mode 100644
... ...
@@ -0,0 +1,41 @@
0
+// +build !appengine
1
+
2
+package msgp
3
+
4
+import (
5
+	"reflect"
6
+	"unsafe"
7
+)
8
+
9
+// NOTE:
10
+// all of the definition in this file
11
+// should be repeated in appengine.go,
12
+// but without using unsafe
13
+
14
+const (
15
+	// spec says int and uint are always
16
+	// the same size, but that int/uint
17
+	// size may not be machine word size
18
+	smallint = unsafe.Sizeof(int(0)) == 4
19
+)
20
+
21
+// UnsafeString returns the byte slice as a volatile string
22
+// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
23
+// THIS IS EVIL CODE.
24
+// YOU HAVE BEEN WARNED.
25
+func UnsafeString(b []byte) string {
26
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
27
+	return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: sh.Data, Len: sh.Len}))
28
+}
29
+
30
+// UnsafeBytes returns the string as a byte slice
31
+// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR.
32
+// THIS IS EVIL CODE.
33
+// YOU HAVE BEEN WARNED.
34
+func UnsafeBytes(s string) []byte {
35
+	return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
36
+		Len:  len(s),
37
+		Cap:  len(s),
38
+		Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data,
39
+	}))
40
+}
... ...
@@ -10,13 +10,6 @@ import (
10 10
 	"time"
11 11
 )
12 12
 
13
-func abs(i int64) int64 {
14
-	if i < 0 {
15
-		return -i
16
-	}
17
-	return i
18
-}
19
-
20 13
 // Sizer is an interface implemented
21 14
 // by types that can estimate their
22 15
 // size when MessagePack encoded.
... ...
@@ -59,15 +52,26 @@ func pushWriter(wr *Writer) {
59 59
 // it will cause undefined behavior.
60 60
 func freeW(w *Writer) { pushWriter(w) }
61 61
 
62
-// Require ensures that cap(old)-len(old) >= extra
62
+// Require ensures that cap(old)-len(old) >= extra.
63 63
 func Require(old []byte, extra int) []byte {
64
-	if cap(old)-len(old) >= extra {
64
+	l := len(old)
65
+	c := cap(old)
66
+	r := l + extra
67
+	if c >= r {
65 68
 		return old
66
-	}
67
-	if len(old) == 0 {
69
+	} else if l == 0 {
68 70
 		return make([]byte, 0, extra)
69 71
 	}
70
-	n := make([]byte, len(old), cap(old)-len(old)+extra)
72
+	// the new size is the greater
73
+	// of double the old capacity
74
+	// and the sum of the old length
75
+	// and the number of new bytes
76
+	// necessary.
77
+	c <<= 1
78
+	if c < r {
79
+		c = r
80
+	}
81
+	n := make([]byte, l, c)
71 82
 	copy(n, old)
72 83
 	return n
73 84
 }
... ...
@@ -184,6 +188,17 @@ func (mw *Writer) require(n int) (int, error) {
184 184
 	return wl, nil
185 185
 }
186 186
 
187
+func (mw *Writer) Append(b ...byte) error {
188
+	if mw.avail() < len(b) {
189
+		err := mw.flush()
190
+		if err != nil {
191
+			return err
192
+		}
193
+	}
194
+	mw.wloc += copy(mw.buf[mw.wloc:], b)
195
+	return nil
196
+}
197
+
187 198
 // push one byte onto the buffer
188 199
 //
189 200
 // NOTE: this is a hot code path
... ...
@@ -289,9 +304,9 @@ func (mw *Writer) Reset(w io.Writer) {
289 289
 // size to the writer
290 290
 func (mw *Writer) WriteMapHeader(sz uint32) error {
291 291
 	switch {
292
-	case sz < 16:
292
+	case sz <= 15:
293 293
 		return mw.push(wfixmap(uint8(sz)))
294
-	case sz < math.MaxUint16:
294
+	case sz <= math.MaxUint16:
295 295
 		return mw.prefix16(mmap16, uint16(sz))
296 296
 	default:
297 297
 		return mw.prefix32(mmap32, sz)
... ...
@@ -302,9 +317,9 @@ func (mw *Writer) WriteMapHeader(sz uint32) error {
302 302
 // given size to the writer
303 303
 func (mw *Writer) WriteArrayHeader(sz uint32) error {
304 304
 	switch {
305
-	case sz < 16:
305
+	case sz <= 15:
306 306
 		return mw.push(wfixarray(uint8(sz)))
307
-	case sz < math.MaxUint16:
307
+	case sz <= math.MaxUint16:
308 308
 		return mw.prefix16(marray16, uint16(sz))
309 309
 	default:
310 310
 		return mw.prefix32(marray32, sz)
... ...
@@ -328,17 +343,26 @@ func (mw *Writer) WriteFloat32(f float32) error {
328 328
 
329 329
 // WriteInt64 writes an int64 to the writer
330 330
 func (mw *Writer) WriteInt64(i int64) error {
331
-	a := abs(i)
331
+	if i >= 0 {
332
+		switch {
333
+		case i <= math.MaxInt8:
334
+			return mw.push(wfixint(uint8(i)))
335
+		case i <= math.MaxInt16:
336
+			return mw.prefix16(mint16, uint16(i))
337
+		case i <= math.MaxInt32:
338
+			return mw.prefix32(mint32, uint32(i))
339
+		default:
340
+			return mw.prefix64(mint64, uint64(i))
341
+		}
342
+	}
332 343
 	switch {
333
-	case i < 0 && i > -32:
344
+	case i >= -32:
334 345
 		return mw.push(wnfixint(int8(i)))
335
-	case i >= 0 && i < 128:
336
-		return mw.push(wfixint(uint8(i)))
337
-	case a < math.MaxInt8:
346
+	case i >= math.MinInt8:
338 347
 		return mw.prefix8(mint8, uint8(i))
339
-	case a < math.MaxInt16:
348
+	case i >= math.MinInt16:
340 349
 		return mw.prefix16(mint16, uint16(i))
341
-	case a < math.MaxInt32:
350
+	case i >= math.MinInt32:
342 351
 		return mw.prefix32(mint32, uint32(i))
343 352
 	default:
344 353
 		return mw.prefix64(mint64, uint64(i))
... ...
@@ -360,20 +384,20 @@ func (mw *Writer) WriteInt(i int) error { return mw.WriteInt64(int64(i)) }
360 360
 // WriteUint64 writes a uint64 to the writer
361 361
 func (mw *Writer) WriteUint64(u uint64) error {
362 362
 	switch {
363
-	case u < (1 << 7):
363
+	case u <= (1<<7)-1:
364 364
 		return mw.push(wfixint(uint8(u)))
365
-	case u < math.MaxUint8:
365
+	case u <= math.MaxUint8:
366 366
 		return mw.prefix8(muint8, uint8(u))
367
-	case u < math.MaxUint16:
367
+	case u <= math.MaxUint16:
368 368
 		return mw.prefix16(muint16, uint16(u))
369
-	case u < math.MaxUint32:
369
+	case u <= math.MaxUint32:
370 370
 		return mw.prefix32(muint32, uint32(u))
371 371
 	default:
372 372
 		return mw.prefix64(muint64, u)
373 373
 	}
374 374
 }
375 375
 
376
-// WriteByte is analagous to WriteUint8
376
+// WriteByte is analogous to WriteUint8
377 377
 func (mw *Writer) WriteByte(u byte) error { return mw.WriteUint8(uint8(u)) }
378 378
 
379 379
 // WriteUint8 writes a uint8 to the writer
... ...
@@ -393,9 +417,9 @@ func (mw *Writer) WriteBytes(b []byte) error {
393 393
 	sz := uint32(len(b))
394 394
 	var err error
395 395
 	switch {
396
-	case sz < math.MaxUint8:
396
+	case sz <= math.MaxUint8:
397 397
 		err = mw.prefix8(mbin8, uint8(sz))
398
-	case sz < math.MaxUint16:
398
+	case sz <= math.MaxUint16:
399 399
 		err = mw.prefix16(mbin16, uint16(sz))
400 400
 	default:
401 401
 		err = mw.prefix32(mbin32, sz)
... ...
@@ -407,6 +431,20 @@ func (mw *Writer) WriteBytes(b []byte) error {
407 407
 	return err
408 408
 }
409 409
 
410
+// WriteBytesHeader writes just the size header
411
+// of a MessagePack 'bin' object. The user is responsible
412
+// for then writing 'sz' more bytes into the stream.
413
+func (mw *Writer) WriteBytesHeader(sz uint32) error {
414
+	switch {
415
+	case sz <= math.MaxUint8:
416
+		return mw.prefix8(mbin8, uint8(sz))
417
+	case sz <= math.MaxUint16:
418
+		return mw.prefix16(mbin16, uint16(sz))
419
+	default:
420
+		return mw.prefix32(mbin32, sz)
421
+	}
422
+}
423
+
410 424
 // WriteBool writes a bool to the writer
411 425
 func (mw *Writer) WriteBool(b bool) error {
412 426
 	if b {
... ...
@@ -421,11 +459,11 @@ func (mw *Writer) WriteString(s string) error {
421 421
 	sz := uint32(len(s))
422 422
 	var err error
423 423
 	switch {
424
-	case sz < 32:
424
+	case sz <= 31:
425 425
 		err = mw.push(wfixstr(uint8(sz)))
426
-	case sz < math.MaxUint8:
426
+	case sz <= math.MaxUint8:
427 427
 		err = mw.prefix8(mstr8, uint8(sz))
428
-	case sz < math.MaxUint16:
428
+	case sz <= math.MaxUint16:
429 429
 		err = mw.prefix16(mstr16, uint16(sz))
430 430
 	default:
431 431
 		err = mw.prefix32(mstr32, sz)
... ...
@@ -436,6 +474,45 @@ func (mw *Writer) WriteString(s string) error {
436 436
 	return mw.writeString(s)
437 437
 }
438 438
 
439
+// WriteStringHeader writes just the string size
440
+// header of a MessagePack 'str' object. The user
441
+// is responsible for writing 'sz' more valid UTF-8
442
+// bytes to the stream.
443
+func (mw *Writer) WriteStringHeader(sz uint32) error {
444
+	switch {
445
+	case sz <= 31:
446
+		return mw.push(wfixstr(uint8(sz)))
447
+	case sz <= math.MaxUint8:
448
+		return mw.prefix8(mstr8, uint8(sz))
449
+	case sz <= math.MaxUint16:
450
+		return mw.prefix16(mstr16, uint16(sz))
451
+	default:
452
+		return mw.prefix32(mstr32, sz)
453
+	}
454
+}
455
+
456
+// WriteStringFromBytes writes a 'str' object
457
+// from a []byte.
458
+func (mw *Writer) WriteStringFromBytes(str []byte) error {
459
+	sz := uint32(len(str))
460
+	var err error
461
+	switch {
462
+	case sz <= 31:
463
+		err = mw.push(wfixstr(uint8(sz)))
464
+	case sz <= math.MaxUint8:
465
+		err = mw.prefix8(mstr8, uint8(sz))
466
+	case sz <= math.MaxUint16:
467
+		err = mw.prefix16(mstr16, uint16(sz))
468
+	default:
469
+		err = mw.prefix32(mstr32, sz)
470
+	}
471
+	if err != nil {
472
+		return err
473
+	}
474
+	_, err = mw.Write(str)
475
+	return err
476
+}
477
+
439 478
 // WriteComplex64 writes a complex64 to the writer
440 479
 func (mw *Writer) WriteComplex64(f complex64) error {
441 480
 	o, err := mw.require(10)
... ...
@@ -509,7 +586,7 @@ func (mw *Writer) WriteMapStrIntf(mp map[string]interface{}) (err error) {
509 509
 // elapsed since "zero" Unix time, followed by 4 bytes
510 510
 // for a big-endian 32-bit signed integer denoting
511 511
 // the nanosecond offset of the time. This encoding
512
-// is intended to ease portability accross languages.
512
+// is intended to ease portability across languages.
513 513
 // (Note that this is *not* the standard time.Time
514 514
 // binary encoding, because its implementation relies
515 515
 // heavily on the internal representation used by the
... ...
@@ -612,7 +689,7 @@ func (mw *Writer) WriteIntf(v interface{}) error {
612 612
 }
613 613
 
614 614
 func (mw *Writer) writeMap(v reflect.Value) (err error) {
615
-	if v.Elem().Kind() != reflect.String {
615
+	if v.Type().Key().Kind() != reflect.String {
616 616
 		return errors.New("msgp: map keys must be strings")
617 617
 	}
618 618
 	ks := v.MapKeys()
... ...
@@ -22,10 +22,10 @@ func ensure(b []byte, sz int) ([]byte, int) {
22 22
 // given size to the slice
23 23
 func AppendMapHeader(b []byte, sz uint32) []byte {
24 24
 	switch {
25
-	case sz < 16:
25
+	case sz <= 15:
26 26
 		return append(b, wfixmap(uint8(sz)))
27 27
 
28
-	case sz < math.MaxUint16:
28
+	case sz <= math.MaxUint16:
29 29
 		o, n := ensure(b, 3)
30 30
 		prefixu16(o[n:], mmap16, uint16(sz))
31 31
 		return o
... ...
@@ -41,10 +41,10 @@ func AppendMapHeader(b []byte, sz uint32) []byte {
41 41
 // the given size to the slice
42 42
 func AppendArrayHeader(b []byte, sz uint32) []byte {
43 43
 	switch {
44
-	case sz < 16:
44
+	case sz <= 15:
45 45
 		return append(b, wfixarray(uint8(sz)))
46 46
 
47
-	case sz < math.MaxUint16:
47
+	case sz <= math.MaxUint16:
48 48
 		o, n := ensure(b, 3)
49 49
 		prefixu16(o[n:], marray16, uint16(sz))
50 50
 		return o
... ...
@@ -75,29 +75,39 @@ func AppendFloat32(b []byte, f float32) []byte {
75 75
 
76 76
 // AppendInt64 appends an int64 to the slice
77 77
 func AppendInt64(b []byte, i int64) []byte {
78
-	a := abs(i)
78
+	if i >= 0 {
79
+		switch {
80
+		case i <= math.MaxInt8:
81
+			return append(b, wfixint(uint8(i)))
82
+		case i <= math.MaxInt16:
83
+			o, n := ensure(b, 3)
84
+			putMint16(o[n:], int16(i))
85
+			return o
86
+		case i <= math.MaxInt32:
87
+			o, n := ensure(b, 5)
88
+			putMint32(o[n:], int32(i))
89
+			return o
90
+		default:
91
+			o, n := ensure(b, 9)
92
+			putMint64(o[n:], i)
93
+			return o
94
+		}
95
+	}
79 96
 	switch {
80
-	case i < 0 && i > -32:
97
+	case i >= -32:
81 98
 		return append(b, wnfixint(int8(i)))
82
-
83
-	case i >= 0 && i < 128:
84
-		return append(b, wfixint(uint8(i)))
85
-
86
-	case a < math.MaxInt8:
99
+	case i >= math.MinInt8:
87 100
 		o, n := ensure(b, 2)
88 101
 		putMint8(o[n:], int8(i))
89 102
 		return o
90
-
91
-	case a < math.MaxInt16:
103
+	case i >= math.MinInt16:
92 104
 		o, n := ensure(b, 3)
93 105
 		putMint16(o[n:], int16(i))
94 106
 		return o
95
-
96
-	case a < math.MaxInt32:
107
+	case i >= math.MinInt32:
97 108
 		o, n := ensure(b, 5)
98 109
 		putMint32(o[n:], int32(i))
99 110
 		return o
100
-
101 111
 	default:
102 112
 		o, n := ensure(b, 9)
103 113
 		putMint64(o[n:], i)
... ...
@@ -120,20 +130,20 @@ func AppendInt32(b []byte, i int32) []byte { return AppendInt64(b, int64(i)) }
120 120
 // AppendUint64 appends a uint64 to the slice
121 121
 func AppendUint64(b []byte, u uint64) []byte {
122 122
 	switch {
123
-	case u < (1 << 7):
123
+	case u <= (1<<7)-1:
124 124
 		return append(b, wfixint(uint8(u)))
125 125
 
126
-	case u < math.MaxUint8:
126
+	case u <= math.MaxUint8:
127 127
 		o, n := ensure(b, 2)
128 128
 		putMuint8(o[n:], uint8(u))
129 129
 		return o
130 130
 
131
-	case u < math.MaxUint16:
131
+	case u <= math.MaxUint16:
132 132
 		o, n := ensure(b, 3)
133 133
 		putMuint16(o[n:], uint16(u))
134 134
 		return o
135 135
 
136
-	case u < math.MaxUint32:
136
+	case u <= math.MaxUint32:
137 137
 		o, n := ensure(b, 5)
138 138
 		putMuint32(o[n:], uint32(u))
139 139
 		return o
... ...
@@ -152,7 +162,7 @@ func AppendUint(b []byte, u uint) []byte { return AppendUint64(b, uint64(u)) }
152 152
 // AppendUint8 appends a uint8 to the slice
153 153
 func AppendUint8(b []byte, u uint8) []byte { return AppendUint64(b, uint64(u)) }
154 154
 
155
-// AppendByte is analagous to AppendUint8
155
+// AppendByte is analogous to AppendUint8
156 156
 func AppendByte(b []byte, u byte) []byte { return AppendUint8(b, uint8(u)) }
157 157
 
158 158
 // AppendUint16 appends a uint16 to the slice
... ...
@@ -167,11 +177,11 @@ func AppendBytes(b []byte, bts []byte) []byte {
167 167
 	var o []byte
168 168
 	var n int
169 169
 	switch {
170
-	case sz < math.MaxUint8:
170
+	case sz <= math.MaxUint8:
171 171
 		o, n = ensure(b, 2+sz)
172 172
 		prefixu8(o[n:], mbin8, uint8(sz))
173 173
 		n += 2
174
-	case sz < math.MaxUint16:
174
+	case sz <= math.MaxUint16:
175 175
 		o, n = ensure(b, 3+sz)
176 176
 		prefixu16(o[n:], mbin16, uint16(sz))
177 177
 		n += 3
... ...
@@ -197,15 +207,15 @@ func AppendString(b []byte, s string) []byte {
197 197
 	var n int
198 198
 	var o []byte
199 199
 	switch {
200
-	case sz < 32:
200
+	case sz <= 31:
201 201
 		o, n = ensure(b, 1+sz)
202 202
 		o[n] = wfixstr(uint8(sz))
203 203
 		n++
204
-	case sz < math.MaxUint8:
204
+	case sz <= math.MaxUint8:
205 205
 		o, n = ensure(b, 2+sz)
206 206
 		prefixu8(o[n:], mstr8, uint8(sz))
207 207
 		n += 2
208
-	case sz < math.MaxUint16:
208
+	case sz <= math.MaxUint16:
209 209
 		o, n = ensure(b, 3+sz)
210 210
 		prefixu16(o[n:], mstr16, uint16(sz))
211 211
 		n += 3
... ...
@@ -217,6 +227,33 @@ func AppendString(b []byte, s string) []byte {
217 217
 	return o[:n+copy(o[n:], s)]
218 218
 }
219 219
 
220
+// AppendStringFromBytes appends a []byte
221
+// as a MessagePack 'str' to the slice 'b.'
222
+func AppendStringFromBytes(b []byte, str []byte) []byte {
223
+	sz := len(str)
224
+	var n int
225
+	var o []byte
226
+	switch {
227
+	case sz <= 31:
228
+		o, n = ensure(b, 1+sz)
229
+		o[n] = wfixstr(uint8(sz))
230
+		n++
231
+	case sz <= math.MaxUint8:
232
+		o, n = ensure(b, 2+sz)
233
+		prefixu8(o[n:], mstr8, uint8(sz))
234
+		n += 2
235
+	case sz <= math.MaxUint16:
236
+		o, n = ensure(b, 3+sz)
237
+		prefixu16(o[n:], mstr16, uint16(sz))
238
+		n += 3
239
+	default:
240
+		o, n = ensure(b, 5+sz)
241
+		prefixu32(o[n:], mstr32, uint32(sz))
242
+		n += 5
243
+	}
244
+	return o[:n+copy(o[n:], str)]
245
+}
246
+
220 247
 // AppendComplex64 appends a complex64 to the slice as a MessagePack extension
221 248
 func AppendComplex64(b []byte, c complex64) []byte {
222 249
 	o, n := ensure(b, Complex64Size)
... ...
@@ -362,7 +399,12 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) {
362 362
 			}
363 363
 		}
364 364
 		return b, nil
365
-
365
+	case reflect.Ptr:
366
+		if v.IsNil() {
367
+			return AppendNil(b), err
368
+		}
369
+		b, err = AppendIntf(b, v.Elem().Interface())
370
+		return b, err
366 371
 	default:
367 372
 		return b, &ErrUnsupportedType{T: v.Type()}
368 373
 	}