Browse code

Implement LZ4 compression.

Implement LZ4 compression, similar to the existing snappy / push-peer-info
model: a LZ4 capable client will send IV_LZ4=1 to the server, and the
algorithm is selected by pushing "compress lz4" back.

LZ4 does not compress as well as LZO or Snappy, but needs far less CPU
and is much faster, thus better suited for mobile devices. See
https://code.google.com/p/lz4/ for more details.

LZ4 include and library path can be specified by specifying LZ4_LIBS=...
and LZ4_CFLAGS=... on the configure command line.

Signed-off-by: Gert Doering <gert@greenie.muc.de>
Acked-by: Arne Schwabe <arne@rfc2549.org>
Message-Id: <1388613479-22377-1-git-send-email-gert@greenie.muc.de>
URL: http://article.gmane.org/gmane.network.openvpn.devel/8153

Gert Doering authored on 2014/01/02 06:57:58
Showing 9 changed files
... ...
@@ -52,6 +52,12 @@ AC_ARG_ENABLE(snappy,
52 52
 	[enable_snappy="yes"]
53 53
 )
54 54
 
55
+AC_ARG_ENABLE(lz4,
56
+	[  --disable-lz4           Disable LZ4 compression support],
57
+	[enable_lz4="$enableval"],
58
+	[enable_lz4="yes"]
59
+)
60
+
55 61
 AC_ARG_ENABLE(comp-stub,
56 62
 	[  --enable-comp-stub      Don't compile compression support but still allow limited interoperability with compression-enabled peers],
57 63
 	[enable_comp_stub="$enableval"],
... ...
@@ -934,6 +940,47 @@ if test "$enable_snappy" = "yes" && test "$enable_comp_stub" = "no"; then
934 934
     CFLAGS="${saved_CFLAGS}"
935 935
 fi
936 936
 
937
+dnl
938
+dnl check for LZ4 library
939
+dnl
940
+
941
+AC_ARG_VAR([LZ4_CFLAGS], [C compiler flags for lz4])
942
+AC_ARG_VAR([LZ4_LIBS], [linker flags for lz4])
943
+if test "$enable_lz4" = "yes" && test "$enable_comp_stub" = "no"; then
944
+    AC_CHECKING([for LZ4 Library and Header files])
945
+    havelz4lib=1
946
+
947
+    # if LZ4_LIBS is set, we assume it will work, otherwise test
948
+    if test -z "${LZ4_LIBS}"; then
949
+	AC_CHECK_LIB(lz4, LZ4_compress,
950
+	    [ LZ4_LIBS="-llz4" ],
951
+	    [
952
+	        AC_MSG_RESULT([LZ4 library not found.])
953
+	        havelz4lib=0
954
+	    ])
955
+    fi
956
+
957
+    saved_CFLAGS="${CFLAGS}"
958
+    CFLAGS="${CFLAGS} ${LZ4_CFLAGS}"
959
+    AC_CHECK_HEADER(lz4.h,
960
+       ,
961
+       [
962
+	   AC_MSG_RESULT([LZ4 headers not found.])
963
+	   havelz4lib=0
964
+       ])
965
+
966
+    if test $havelz4lib = 0 ; then
967
+	AC_MSG_RESULT([LZ4 library available from http://code.google.com/p/lz4/])
968
+        AC_MSG_ERROR([Or try ./configure --disable-lz4 OR ./configure --enable-comp-stub])
969
+    fi
970
+    OPTIONAL_LZ4_CFLAGS="${LZ4_CFLAGS}"
971
+    OPTIONAL_LZ4_LIBS="${LZ4_LIBS}"
972
+    AC_DEFINE(ENABLE_LZ4, 1, [Enable LZ4 compression library])
973
+    CFLAGS="${saved_CFLAGS}"
974
+fi
975
+
976
+
977
+
937 978
 
938 979
 AC_MSG_CHECKING([git checkout])
939 980
 GIT_CHECKOUT="no"
... ...
@@ -1045,6 +1092,7 @@ fi
1045 1045
 if test "${enable_comp_stub}" = "yes"; then
1046 1046
 	test "${enable_lzo}" = "yes" && AC_MSG_ERROR([Cannot have both comp stub and lzo enabled (use --disable-lzo)])
1047 1047
 	test "${enable_snappy}" = "yes" && AC_MSG_ERROR([Cannot have both comp stub and snappy enabled (use --disable-snappy)])
1048
+	test "${enable_lz4}" = "yes" && AC_MSG_ERROR([Cannot have both comp stub and LZ4 enabled (use --disable-lz4)])
1048 1049
 	AC_DEFINE([ENABLE_COMP_STUB], [1], [Enable compression stub capability])
1049 1050
 fi
1050 1051
 
... ...
@@ -1101,6 +1149,8 @@ AC_SUBST([OPTIONAL_LZO_CFLAGS])
1101 1101
 AC_SUBST([OPTIONAL_LZO_LIBS])
1102 1102
 AC_SUBST([OPTIONAL_SNAPPY_CFLAGS])
1103 1103
 AC_SUBST([OPTIONAL_SNAPPY_LIBS])
1104
+AC_SUBST([OPTIONAL_LZ4_CFLAGS])
1105
+AC_SUBST([OPTIONAL_LZ4_LIBS])
1104 1106
 AC_SUBST([OPTIONAL_PKCS11_HELPER_CFLAGS])
1105 1107
 AC_SUBST([OPTIONAL_PKCS11_HELPER_LIBS])
1106 1108
 
... ...
@@ -27,6 +27,7 @@ AM_CFLAGS = \
27 27
 	$(OPTIONAL_CRYPTO_CFLAGS) \
28 28
 	$(OPTIONAL_LZO_CFLAGS) \
29 29
 	$(OPTIONAL_SNAPPY_CFLAGS) \
30
+	$(OPTIONAL_LZ4_CFLAGS) \
30 31
 	$(OPTIONAL_PKCS11_HELPER_CFLAGS)
31 32
 if WIN32
32 33
 # we want unicode entry point but not the macro
... ...
@@ -43,6 +44,7 @@ openvpn_SOURCES = \
43 43
 	clinat.c clinat.h \
44 44
 	common.h \
45 45
 	comp.c comp.h compstub.c \
46
+	comp-lz4.c comp-lz4.h \
46 47
 	crypto.c crypto.h crypto_backend.h \
47 48
 	crypto_openssl.c crypto_openssl.h \
48 49
 	crypto_polarssl.c crypto_polarssl.h \
... ...
@@ -120,6 +122,7 @@ openvpn_LDADD = \
120 120
 	$(SOCKETS_LIBS) \
121 121
 	$(OPTIONAL_LZO_LIBS) \
122 122
 	$(OPTIONAL_SNAPPY_LIBS) \
123
+	$(OPTIONAL_LZ4_LIBS) \
123 124
 	$(OPTIONAL_PKCS11_HELPER_LIBS) \
124 125
 	$(OPTIONAL_CRYPTO_LIBS) \
125 126
 	$(OPTIONAL_SELINUX_LIBS) \
126 127
new file mode 100644
... ...
@@ -0,0 +1,191 @@
0
+/*
1
+ *  OpenVPN -- An application to securely tunnel IP networks
2
+ *             over a single UDP port, with support for SSL/TLS-based
3
+ *             session authentication and key exchange,
4
+ *             packet encryption, packet authentication, and
5
+ *             packet compression.
6
+ *
7
+ *  Copyright (C) 2002-2012 OpenVPN Technologies, Inc. <sales@openvpn.net>
8
+ *  Copyright (C) 2013      Gert Doering <gert@greenie.muc.de>
9
+ *
10
+ *  This program is free software; you can redistribute it and/or modify
11
+ *  it under the terms of the GNU General Public License version 2
12
+ *  as published by the Free Software Foundation.
13
+ *
14
+ *  This program is distributed in the hope that it will be useful,
15
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
+ *  GNU General Public License for more details.
18
+ *
19
+ *  You should have received a copy of the GNU General Public License
20
+ *  along with this program (see the file COPYING included with this
21
+ *  distribution); if not, write to the Free Software Foundation, Inc.,
22
+ *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
23
+ */
24
+
25
+#ifdef HAVE_CONFIG_H
26
+#include "config.h"
27
+#elif defined(_MSC_VER)
28
+#include "config-msvc.h"
29
+#endif
30
+
31
+#include "syshead.h"
32
+
33
+#if defined(ENABLE_LZ4)
34
+
35
+#include "lz4.h"
36
+
37
+#include "comp.h"
38
+#include "error.h"
39
+
40
+#include "memdbg.h"
41
+
42
+/* Initial command byte to tell our peer if we compressed */
43
+#define LZ4_COMPRESS_BYTE 0x69
44
+
45
+static void
46
+lz4_compress_init (struct compress_context *compctx)
47
+{
48
+  msg (D_INIT_MEDIUM, "LZ4 compression initializing");
49
+  ASSERT(compctx->flags & COMP_F_SWAP);
50
+}
51
+
52
+static void
53
+lz4_compress_uninit (struct compress_context *compctx)
54
+{
55
+}
56
+
57
+static void
58
+lz4_compress (struct buffer *buf, struct buffer work,
59
+	       struct compress_context *compctx,
60
+	       const struct frame* frame)
61
+{
62
+  int result;
63
+  bool compressed = false;
64
+
65
+  if (buf->len <= 0)
66
+    return;
67
+
68
+  /*
69
+   * In order to attempt compression, length must be at least COMPRESS_THRESHOLD.
70
+   */
71
+  if (buf->len >= COMPRESS_THRESHOLD)
72
+    {
73
+      const size_t ps = PAYLOAD_SIZE (frame);
74
+      int zlen_max = ps + COMP_EXTRA_BUFFER (ps);
75
+      int zlen;
76
+
77
+      ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
78
+      ASSERT (buf_safe (&work, zlen_max));
79
+
80
+      if (buf->len > ps)
81
+	{
82
+	  dmsg (D_COMP_ERRORS, "LZ4 compression buffer overflow");
83
+	  buf->len = 0;
84
+	  return;
85
+	}
86
+
87
+      zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char *)BPTR(&work), BLEN(buf), zlen_max );
88
+
89
+      if (zlen <= 0)
90
+	{
91
+	  dmsg (D_COMP_ERRORS, "LZ4 compression error");
92
+	  buf->len = 0;
93
+	  return;
94
+	}
95
+
96
+      ASSERT (buf_safe (&work, zlen));
97
+      work.len = zlen;
98
+      compressed = true;
99
+
100
+      dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work.len);
101
+      compctx->pre_compress += buf->len;
102
+      compctx->post_compress += work.len;
103
+    }
104
+
105
+  /* did compression save us anything? */
106
+  {
107
+    uint8_t comp_head_byte = NO_COMPRESS_BYTE_SWAP;
108
+    if (compressed && work.len < buf->len)
109
+      {
110
+	*buf = work;
111
+	comp_head_byte = LZ4_COMPRESS_BYTE;
112
+      }
113
+
114
+    {
115
+      uint8_t *head = BPTR (buf);
116
+      uint8_t *tail  = BEND (buf);
117
+      ASSERT (buf_safe (buf, 1));
118
+      ++buf->len;
119
+
120
+      /* move head byte of payload to tail */
121
+      *tail = *head;
122
+      *head = comp_head_byte;
123
+    }
124
+  }
125
+}
126
+
127
+static void
128
+lz4_decompress (struct buffer *buf, struct buffer work,
129
+		 struct compress_context *compctx,
130
+		 const struct frame* frame)
131
+{
132
+  size_t zlen_max = EXPANDED_SIZE (frame);
133
+  int uncomp_len;
134
+  uint8_t c;		/* flag indicating whether or not our peer compressed */
135
+
136
+  if (buf->len <= 0)
137
+    return;
138
+
139
+  ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
140
+
141
+  /* do unframing/swap (assumes buf->len > 0) */
142
+  {
143
+    uint8_t *head = BPTR (buf);
144
+    c = *head;
145
+    --buf->len;
146
+    *head = *BEND (buf);
147
+  }
148
+
149
+  if (c == LZ4_COMPRESS_BYTE)	/* packet was compressed */
150
+    {
151
+      ASSERT (buf_safe (&work, zlen_max));
152
+      uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char *)BPTR(&work), (size_t)BLEN(buf), zlen_max);
153
+      if (uncomp_len <= 0)
154
+	{
155
+	  dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
156
+	  buf->len = 0;
157
+	  return;
158
+	}
159
+
160
+      ASSERT (buf_safe (&work, uncomp_len));
161
+      work.len = uncomp_len;
162
+
163
+      dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work.len);
164
+      compctx->pre_decompress += buf->len;
165
+      compctx->post_decompress += work.len;
166
+
167
+      *buf = work;
168
+    }
169
+  else if (c == NO_COMPRESS_BYTE_SWAP)	/* packet was not compressed */
170
+    {
171
+      ;
172
+    }
173
+  else
174
+    {
175
+      dmsg (D_COMP_ERRORS, "Bad LZ4 decompression header byte: %d", c);
176
+      buf->len = 0;
177
+    }
178
+}
179
+
180
+const struct compress_alg lz4_alg = {
181
+  "lz4",
182
+  lz4_compress_init,
183
+  lz4_compress_uninit,
184
+  lz4_compress,
185
+  lz4_decompress
186
+};
187
+
188
+#else
189
+static void dummy(void) {}
190
+#endif /* ENABLE_LZ4 */
0 191
new file mode 100644
... ...
@@ -0,0 +1,40 @@
0
+/*
1
+ *  OpenVPN -- An application to securely tunnel IP networks
2
+ *             over a single UDP port, with support for SSL/TLS-based
3
+ *             session authentication and key exchange,
4
+ *             packet encryption, packet authentication, and
5
+ *             packet compression.
6
+ *
7
+ *  Copyright (C) 2002-2012 OpenVPN Technologies, Inc. <sales@openvpn.net>
8
+ *  Copyright (C) 2013      Gert Doering <gert@greenie.muc.de>
9
+ *
10
+ *  This program is free software; you can redistribute it and/or modify
11
+ *  it under the terms of the GNU General Public License version 2
12
+ *  as published by the Free Software Foundation.
13
+ *
14
+ *  This program is distributed in the hope that it will be useful,
15
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
+ *  GNU General Public License for more details.
18
+ *
19
+ *  You should have received a copy of the GNU General Public License
20
+ *  along with this program (see the file COPYING included with this
21
+ *  distribution); if not, write to the Free Software Foundation, Inc.,
22
+ *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
23
+ */
24
+
25
+#ifndef OPENVPN_COMP_LZ4_H
26
+#define OPENVPN_COMP_LZ4_H
27
+
28
+#if defined(ENABLE_LZ4)
29
+
30
+#include "buffer.h"
31
+
32
+extern const struct compress_alg lz4_alg;
33
+
34
+struct lz4_workspace
35
+{
36
+};
37
+
38
+#endif /* ENABLE_LZ4 */
39
+#endif
... ...
@@ -66,6 +66,14 @@ comp_init(const struct compress_options *opt)
66 66
       (*compctx->alg.compress_init)(compctx);
67 67
       break;
68 68
 #endif
69
+#ifdef ENABLE_LZ4
70
+    case COMP_ALG_LZ4:
71
+      ALLOC_OBJ_CLEAR (compctx, struct compress_context);
72
+      compctx->flags = opt->flags;
73
+      compctx->alg = lz4_alg;
74
+      (*compctx->alg.compress_init)(compctx);
75
+      break;
76
+#endif
69 77
     }
70 78
   return compctx;
71 79
 }
... ...
@@ -118,6 +126,9 @@ comp_generate_peer_info_string(const struct compress_options *opt, struct buffer
118 118
       bool lzo_avail = false;
119 119
       if (!(opt->flags & COMP_F_ADVERTISE_STUBS_ONLY))
120 120
 	{
121
+#if defined(ENABLE_LZ4)
122
+	  buf_printf (out, "IV_LZ4=1\n");
123
+#endif
121 124
 #if defined(ENABLE_SNAPPY)
122 125
 	  buf_printf (out, "IV_SNAPPY=1\n");
123 126
 #endif
... ...
@@ -24,7 +24,7 @@
24 24
 
25 25
 /*
26 26
  * Generic compression support.  Currently we support
27
- * Snappy and LZO 2.
27
+ * Snappy, LZO 2 and LZ4.
28 28
  */
29 29
 #ifndef OPENVPN_COMP_H
30 30
 #define OPENVPN_COMP_H
... ...
@@ -41,6 +41,7 @@
41 41
 #define COMP_ALG_STUB   1 /* support compression command byte and framing without actual compression */
42 42
 #define COMP_ALG_LZO    2 /* LZO algorithm */
43 43
 #define COMP_ALG_SNAPPY 3 /* Snappy algorithm */
44
+#define COMP_ALG_LZ4    4 /* LZ4 algorithm */
44 45
 
45 46
 /* Compression flags */
46 47
 #define COMP_F_ADAPTIVE   (1<<0) /* COMP_ALG_LZO only */
... ...
@@ -64,6 +65,7 @@
64 64
  *
65 65
  * LZO:    len + len/8 + 128 + 3
66 66
  * Snappy: len + len/6 + 32
67
+ * LZ4:    len + len/255 + 16  (LZ4_COMPRESSBOUND(len))
67 68
  */
68 69
 #define COMP_EXTRA_BUFFER(len) ((len)/6 + 128 + 3 + COMP_PREFIX_LEN)
69 70
 
... ...
@@ -103,6 +105,10 @@ struct compress_alg
103 103
 #include "snappy.h"
104 104
 #endif
105 105
 
106
+#ifdef ENABLE_LZ4
107
+#include "comp-lz4.h"
108
+#endif
109
+
106 110
 /*
107 111
  * Information that basically identifies a compression
108 112
  * algorithm and related flags.
... ...
@@ -124,6 +130,9 @@ union compress_workspace_union
124 124
 #ifdef ENABLE_SNAPPY
125 125
   struct snappy_workspace snappy;
126 126
 #endif
127
+#ifdef ENABLE_LZ4
128
+  struct lz4_workspace lz4;
129
+#endif
127 130
 };
128 131
 
129 132
 /*
... ...
@@ -2379,7 +2379,7 @@ do_init_frame (struct context *c)
2379 2379
     {
2380 2380
       comp_add_to_extra_frame (&c->c2.frame);
2381 2381
 
2382
-#if !defined(ENABLE_SNAPPY)
2382
+#if !defined(ENABLE_SNAPPY) && !defined(ENABLE_LZ4)
2383 2383
       /*
2384 2384
        * Compression usage affects buffer alignment when non-swapped algs
2385 2385
        * such as LZO is used.
... ...
@@ -2394,7 +2394,7 @@ do_init_frame (struct context *c)
2394 2394
        * dispatch if packet is uncompressed) at the cost of requiring
2395 2395
        * decryption output to be written to an unaligned buffer, so
2396 2396
        * it's more of a tradeoff than an optimal solution and we don't
2397
-       * include it when we are doing a modern build with Snappy.
2397
+       * include it when we are doing a modern build with Snappy or LZ4.
2398 2398
        * Strictly speaking, on the server it would be better to execute
2399 2399
        * this code for every connection after we decide the compression
2400 2400
        * method, but currently the frame code doesn't appear to be
... ...
@@ -92,6 +92,9 @@ const char title_string[] =
92 92
 #ifdef ENABLE_SNAPPY
93 93
   " [SNAPPY]"
94 94
 #endif
95
+#ifdef ENABLE_LZ4
96
+  " [LZ4]"
97
+#endif
95 98
 #ifdef ENABLE_COMP_STUB
96 99
   " [COMP_STUB]"
97 100
 #endif
... ...
@@ -6259,6 +6262,13 @@ add_option (struct options *options,
6259 6259
 	      options->comp.flags = COMP_F_SWAP;
6260 6260
 	    }
6261 6261
 #endif
6262
+#if defined(ENABLE_LZ4)
6263
+	  else if (streq (p[1], "lz4"))
6264
+	    {
6265
+	      options->comp.alg = COMP_ALG_LZ4;
6266
+	      options->comp.flags = COMP_F_SWAP;
6267
+	    }
6268
+#endif
6262 6269
 	  else
6263 6270
 	    {
6264 6271
 	      msg (msglevel, "bad comp option: %s", p[1]);
... ...
@@ -715,7 +715,8 @@ socket_defined (const socket_descriptor_t sd)
715 715
 /*
716 716
  * Compression support
717 717
  */
718
-#if defined(ENABLE_SNAPPY) || defined(ENABLE_LZO) || defined(ENABLE_COMP_STUB)
718
+#if defined(ENABLE_SNAPPY) || defined(ENABLE_LZO) || defined(ENABLE_LZ4) || \
719
+    defined(ENABLE_COMP_STUB)
719 720
 #define USE_COMP
720 721
 #endif
721 722