Browse code

add gelf option to customize compression type and level

this allows user to choose the compression type (i.e. gzip/zlib/none) using
--log-opt=gelf-compression-type=none or the compression level (-1..9) using
--log-opt=gelf-compression-level=0 for gelf driver.

Signed-off-by: Daniel Dao <dqminh@cloudflare.com>

Daniel Dao authored on 2016/01/21 21:44:25
Showing 2 changed files
... ...
@@ -6,9 +6,12 @@ package gelf
6 6
 
7 7
 import (
8 8
 	"bytes"
9
+	"compress/flate"
10
+	"encoding/json"
9 11
 	"fmt"
10 12
 	"net"
11 13
 	"net/url"
14
+	"strconv"
12 15
 	"time"
13 16
 
14 17
 	"github.com/Graylog2/go-gelf/gelf"
... ...
@@ -24,7 +27,7 @@ type gelfLogger struct {
24 24
 	writer   *gelf.Writer
25 25
 	ctx      logger.Context
26 26
 	hostname string
27
-	extra    map[string]interface{}
27
+	rawExtra json.RawMessage
28 28
 }
29 29
 
30 30
 func init() {
... ...
@@ -81,17 +84,43 @@ func New(ctx logger.Context) (logger.Logger, error) {
81 81
 		extra[k] = v
82 82
 	}
83 83
 
84
+	rawExtra, err := json.Marshal(extra)
85
+	if err != nil {
86
+		return nil, err
87
+	}
88
+
84 89
 	// create new gelfWriter
85 90
 	gelfWriter, err := gelf.NewWriter(address)
86 91
 	if err != nil {
87 92
 		return nil, fmt.Errorf("gelf: cannot connect to GELF endpoint: %s %v", address, err)
88 93
 	}
89 94
 
95
+	if v, ok := ctx.Config["gelf-compression-type"]; ok {
96
+		switch v {
97
+		case "gzip":
98
+			gelfWriter.CompressionType = gelf.CompressGzip
99
+		case "zlib":
100
+			gelfWriter.CompressionType = gelf.CompressZlib
101
+		case "none":
102
+			gelfWriter.CompressionType = gelf.CompressNone
103
+		default:
104
+			return nil, fmt.Errorf("gelf: invalid compression type %q", v)
105
+		}
106
+	}
107
+
108
+	if v, ok := ctx.Config["gelf-compression-level"]; ok {
109
+		val, err := strconv.Atoi(v)
110
+		if err != nil {
111
+			return nil, fmt.Errorf("gelf: invalid compression level %s, err %v", v, err)
112
+		}
113
+		gelfWriter.CompressionLevel = val
114
+	}
115
+
90 116
 	return &gelfLogger{
91 117
 		writer:   gelfWriter,
92 118
 		ctx:      ctx,
93 119
 		hostname: hostname,
94
-		extra:    extra,
120
+		rawExtra: rawExtra,
95 121
 	}, nil
96 122
 }
97 123
 
... ...
@@ -107,7 +136,7 @@ func (s *gelfLogger) Log(msg *logger.Message) error {
107 107
 		Short:    string(msg.Line),
108 108
 		TimeUnix: float64(msg.Timestamp.UnixNano()/int64(time.Millisecond)) / 1000.0,
109 109
 		Level:    level,
110
-		Extra:    s.extra,
110
+		RawExtra: s.rawExtra,
111 111
 	}
112 112
 
113 113
 	if err := s.writer.WriteMessage(&m); err != nil {
... ...
@@ -127,15 +156,26 @@ func (s *gelfLogger) Name() string {
127 127
 // ValidateLogOpt looks for gelf specific log options gelf-address, &
128 128
 // gelf-tag.
129 129
 func ValidateLogOpt(cfg map[string]string) error {
130
-	for key := range cfg {
130
+	for key, val := range cfg {
131 131
 		switch key {
132 132
 		case "gelf-address":
133 133
 		case "gelf-tag":
134 134
 		case "tag":
135 135
 		case "labels":
136 136
 		case "env":
137
+		case "gelf-compression-level":
138
+			i, err := strconv.Atoi(val)
139
+			if err != nil || i < flate.DefaultCompression || i > flate.BestCompression {
140
+				return fmt.Errorf("unknown value %q for log opt %q for gelf log driver", val, key)
141
+			}
142
+		case "gelf-compression-type":
143
+			switch val {
144
+			case "gzip", "zlib", "none":
145
+			default:
146
+				return fmt.Errorf("unknown value %q for log opt %q for gelf log driver", val, key)
147
+			}
137 148
 		default:
138
-			return fmt.Errorf("unknown log opt '%s' for gelf log driver", key)
149
+			return fmt.Errorf("unknown log opt %q for gelf log driver", key)
139 150
 		}
140 151
 	}
141 152
 
... ...
@@ -146,6 +146,8 @@ The GELF logging driver supports the following options:
146 146
     --log-opt tag="database"
147 147
     --log-opt labels=label1,label2
148 148
     --log-opt env=env1,env2
149
+    --log-opt gelf-compression-type=gzip
150
+    --log-opt gelf-compression-level=1
149 151
 
150 152
 The `gelf-address` option specifies the remote GELF server address that the
151 153
 driver connects to. Currently, only `udp` is supported as the transport and you must
... ...
@@ -167,6 +169,14 @@ underscore (`_`).
167 167
     "_fizz": "buzz",
168 168
     // […]
169 169
 
170
+The `gelf-compression-type` option can be used to change how the GELF driver
171
+compresses each log message. The accepted values are `gzip`, `zlib` and `none`.
172
+`gzip` is chosen by default.
173
+
174
+The `gelf-compression-level` option can be used to change the level of compresssion
175
+when `gzip` or `zlib` is selected as `gelf-compression-type`. Accepted value
176
+must be from from -1 to 9 (BestCompression). Higher levels typically
177
+run slower but compress more. Default value is 1 (BestSpeed).
170 178
 
171 179
 ## fluentd options
172 180