diff --git a/net/haproxy/Makefile b/net/haproxy/Makefile index e01785a972..c99c9eabff 100644 --- a/net/haproxy/Makefile +++ b/net/haproxy/Makefile @@ -11,7 +11,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=haproxy PKG_VERSION:=1.8.19 -PKG_RELEASE:=3 +PKG_RELEASE:=4 PKG_SOURCE:=haproxy-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://www.haproxy.org/download/1.8/src/ diff --git a/net/haproxy/patches/008-BUG-MAJOR-stats-Fix-how-huge-POST-data-are-read-from-the-channel.patch b/net/haproxy/patches/008-BUG-MAJOR-stats-Fix-how-huge-POST-data-are-read-from-the-channel.patch new file mode 100644 index 0000000000..2b293e4779 --- /dev/null +++ b/net/haproxy/patches/008-BUG-MAJOR-stats-Fix-how-huge-POST-data-are-read-from-the-channel.patch @@ -0,0 +1,92 @@ +commit 3c6ad99924236bf8b7741030bd163aacb820d451 +Author: Christopher Faulet +Date: Wed Feb 27 15:30:57 2019 +0100 + + BUG/MAJOR: stats: Fix how huge POST data are read from the channel + + When the body length is greater than a chunk size (so if length of POST data + exceeds the buffer size), the requests is rejected with the status code + STAT_STATUS_EXCD. Otherwise the stats applet will wait to have all the data to + copy and parse them. But there is a problem when the total request size + (including the headers) is just lower than the buffer size but greater the + buffer size less the reserve. In such case, the body length is considered as + enough small to be processed but not entierly received. So the stats applet + waits for more data. But because outgoing data are still there, the channel's + buffer is considered as full and nothing more can be read, leading to a freeze + of the session. + + Note this bug is pretty easy to reproduce with the legacy HTTP. It is harder + with the HTX but still possible. To fix the bug, in the stats applet, when the + request is not fully received, we check if at least the reserve remains + available the channel's buffer. + + This patch must be backported as far as 1.5. But because the HTX does not exist + in 1.8 and lower, it will have to be adapted for these versions. + + (cherry picked from commit 2f9a41d52b28b88d44aab063bb2b88025a388981) + Signed-off-by: Willy Tarreau + (cherry picked from commit 18a5b1eece7e12047c1dc371c42f72d9d129ce0a) + [cf: HTX code removed and calls to the new buffer API replaced to use the old + one] + Signed-off-by: Christopher Faulet + +diff --git a/src/stats.c b/src/stats.c +index 4973bb8a..0a6c15f6 100644 +--- a/src/stats.c ++++ b/src/stats.c +@@ -2441,6 +2441,7 @@ static void stats_dump_html_info(struct stream_interface *si, struct uri_auth *u + "Action not processed because of invalid parameters." + "
    " + "
  • The action is maybe unknown.
    • " ++ "
  • Invalid key parameter (empty or too long).
    • " + "
  • The backend name is probably unknown or ambiguous (duplicated names).
    • " + "
  • Some server names are probably unknown or ambiguous (duplicated names in the backend).
    • " + "
" +@@ -2634,17 +2635,20 @@ static int stats_process_http_post(struct stream_interface *si) + int reql; + + temp = get_trash_chunk(); +- if (temp->size < s->txn->req.body_len) { +- /* too large request */ +- appctx->ctx.stats.st_code = STAT_STATUS_EXCD; +- goto out; +- } + ++ /* we need more data */ ++ if (s->txn->req.msg_state < HTTP_MSG_DONE) { ++ /* check if we can receive more */ ++ if (buffer_total_space(s->req.buf) <= global.tune.maxrewrite) { ++ appctx->ctx.stats.st_code = STAT_STATUS_EXCD; ++ goto out; ++ } ++ goto wait; ++ } + reql = co_getblk(si_oc(si), temp->str, s->txn->req.body_len, s->txn->req.eoh + 2); + if (reql <= 0) { +- /* we need more data */ +- appctx->ctx.stats.st_code = STAT_STATUS_NONE; +- return 0; ++ appctx->ctx.stats.st_code = STAT_STATUS_EXCD; ++ goto out; + } + + first_param = temp->str; +@@ -2673,7 +2677,7 @@ static int stats_process_http_post(struct stream_interface *si) + strncpy(key, cur_param + poffset, plen); + key[plen - 1] = '\0'; + } else { +- appctx->ctx.stats.st_code = STAT_STATUS_EXCD; ++ appctx->ctx.stats.st_code = STAT_STATUS_ERRP; + goto out; + } + +@@ -2929,6 +2933,9 @@ static int stats_process_http_post(struct stream_interface *si) + } + out: + return 1; ++ wait: ++ appctx->ctx.stats.st_code = STAT_STATUS_NONE; ++ return 0; + } + + diff --git a/net/haproxy/patches/009-BUG-MINOR-http-counters-fix-missing-increment-of-fe--srv_aborts.patch b/net/haproxy/patches/009-BUG-MINOR-http-counters-fix-missing-increment-of-fe--srv_aborts.patch new file mode 100644 index 0000000000..ae1a0e553a --- /dev/null +++ b/net/haproxy/patches/009-BUG-MINOR-http-counters-fix-missing-increment-of-fe--srv_aborts.patch @@ -0,0 +1,30 @@ +commit f5ed1f24a9f7bb3b58f6f29b403963e0155e44a9 +Author: Willy Tarreau +Date: Mon Mar 18 11:02:57 2019 +0100 + + BUG/MINOR: http/counters: fix missing increment of fe->srv_aborts + + When a server aborts a transfer, we used to increment the backend's + counter but not the frontend's during the forwarding phase. This fixes + it. It might be backported to all supported versions (possibly removing + the htx part) though it is of very low importance. + + (cherry picked from commit d1fd6f5f64e4d05d4993f2d43c1ee8c79a16fec1) + [wt: s/_HA_ATOMIC/HA_ATOMIC/] + Signed-off-by: Willy Tarreau + (cherry picked from commit 8aa5c6f660ecc9ed79e759a70112e1dbfd59831d) + [cf: HTX code removed] + Signed-off-by: Christopher Faulet + +diff --git a/src/proto_http.c b/src/proto_http.c +index efd318e7..8b087c5b 100644 +--- a/src/proto_http.c ++++ b/src/proto_http.c +@@ -6150,6 +6150,7 @@ int http_response_forward_body(struct stream *s, struct channel *res, int an_bit + if (!buffer_pending(res->buf)) { + if (!(s->flags & SF_ERR_MASK)) + s->flags |= SF_ERR_SRVCL; ++ HA_ATOMIC_ADD(&sess->fe->fe_counters.srv_aborts, 1); + HA_ATOMIC_ADD(&s->be->be_counters.srv_aborts, 1); + if (objt_server(s->target)) + HA_ATOMIC_ADD(&objt_server(s->target)->counters.srv_aborts, 1); diff --git a/net/haproxy/patches/010-BUG-MEDIUM-ssl-ability-to-set-TLS-1-3-ciphers-using-ssl-default-server-ciphersuites.patch b/net/haproxy/patches/010-BUG-MEDIUM-ssl-ability-to-set-TLS-1-3-ciphers-using-ssl-default-server-ciphersuites.patch new file mode 100644 index 0000000000..0fb3c0a910 --- /dev/null +++ b/net/haproxy/patches/010-BUG-MEDIUM-ssl-ability-to-set-TLS-1-3-ciphers-using-ssl-default-server-ciphersuites.patch @@ -0,0 +1,36 @@ +commit a2919cab08fff3fad434c574506b5ae23b4db131 +Author: Pierre Cheynier +Date: Thu Mar 21 16:15:47 2019 +0000 + + BUG/MEDIUM: ssl: ability to set TLS 1.3 ciphers using ssl-default-server-ciphersuites + + Any attempt to put TLS 1.3 ciphers on servers failed with output 'unable + to set TLS 1.3 cipher suites'. + + This was due to usage of SSL_CTX_set_cipher_list instead of + SSL_CTX_set_ciphersuites in the TLS 1.3 block (protected by + OPENSSL_VERSION_NUMBER >= 0x10101000L & so). + + This should be backported to 1.9 and 1.8. + + Signed-off-by: Pierre Cheynier + Reported-by: Damien Claisse + Cc: Emeric Brun + (cherry picked from commit bc34cd1de2ee80de63b5c4d319a501fc0d4ea2f5) + Signed-off-by: Willy Tarreau + (cherry picked from commit 4a8b9e3d5f4e76295c571900771fd1728bec474f) + Signed-off-by: Christopher Faulet + +diff --git a/src/ssl_sock.c b/src/ssl_sock.c +index afdb1fce..fbb7cf2b 100644 +--- a/src/ssl_sock.c ++++ b/src/ssl_sock.c +@@ -4696,7 +4696,7 @@ int ssl_sock_prepare_srv_ctx(struct server *srv) + + #if (OPENSSL_VERSION_NUMBER >= 0x10101000L && !defined OPENSSL_IS_BORINGSSL && !defined LIBRESSL_VERSION_NUMBER) + if (srv->ssl_ctx.ciphersuites && +- !SSL_CTX_set_cipher_list(srv->ssl_ctx.ctx, srv->ssl_ctx.ciphersuites)) { ++ !SSL_CTX_set_ciphersuites(srv->ssl_ctx.ctx, srv->ssl_ctx.ciphersuites)) { + ha_alert("Proxy '%s', server '%s' [%s:%d] : unable to set TLS 1.3 cipher suites to '%s'.\n", + curproxy->id, srv->id, + srv->conf.file, srv->conf.line, srv->ssl_ctx.ciphersuites); diff --git a/net/haproxy/patches/011-DOC-The-option-httplog-is-no-longer-valid-in-a-backend.patch b/net/haproxy/patches/011-DOC-The-option-httplog-is-no-longer-valid-in-a-backend.patch new file mode 100644 index 0000000000..9d913dda17 --- /dev/null +++ b/net/haproxy/patches/011-DOC-The-option-httplog-is-no-longer-valid-in-a-backend.patch @@ -0,0 +1,26 @@ +commit 6e9787fcc677b15cb9fc0baea8de545127b4fb85 +Author: Freddy Spierenburg +Date: Mon Mar 25 14:35:17 2019 +0100 + + DOC: The option httplog is no longer valid in a backend. + + This can be backported to 1.9 and 1.8. + + (cherry picked from commit e88b77351b3fe34a83d5208773a63f78fb140722) + Signed-off-by: Willy Tarreau + (cherry picked from commit 79b70b0f8f86315230323e7f826796dffba4b85b) + Signed-off-by: Christopher Faulet + +diff --git a/doc/configuration.txt b/doc/configuration.txt +index 8d75d568..3608a897 100644 +--- a/doc/configuration.txt ++++ b/doc/configuration.txt +@@ -2118,7 +2118,7 @@ option http-tunnel (*) X X X X + option http-use-proxy-header (*) X X X - + option httpchk X - X X + option httpclose (*) X X X X +-option httplog X X X X ++option httplog X X X - + option http_proxy (*) X X X X + option independent-streams (*) X X X X + option ldap-check X - X X diff --git a/net/haproxy/patches/012-BUG-MAJOR-checks-segfault-during-tcpcheck_main.patch b/net/haproxy/patches/012-BUG-MAJOR-checks-segfault-during-tcpcheck_main.patch new file mode 100644 index 0000000000..4d9fc029e9 --- /dev/null +++ b/net/haproxy/patches/012-BUG-MAJOR-checks-segfault-during-tcpcheck_main.patch @@ -0,0 +1,40 @@ +commit ed3951cf6d9c7846fc780042fdddc194dda47c8d +Author: Ricardo Nabinger Sanchez +Date: Thu Mar 28 21:42:23 2019 -0300 + + BUG/MAJOR: checks: segfault during tcpcheck_main + + When using TCP health checks (tcp-check connect), it is possible to + crash with a segfault when, for reasons yet to be understood, the + protocol family is unknown. + + In the function tcpcheck_main(), proto is dereferenced without a prior + test in case it is NULL, leading to the segfault during proto->connect + dereference. + + The line has been unmodified since it was introduced, in commit + 69e273f3fcfbfb9cc0fb5a09668faad66cfbd36b. This was the only use of + proto (or more specifically, the return of protocol_by_family()) that + was unprotected; all other callsites perform the test for a NULL + pointer. + + This patch should be backported to 1.9, 1.8, 1.7, and 1.6. + + (cherry picked from commit 4bccea98912c74fa42c665ec25e417c2cca4eee7) + Signed-off-by: Willy Tarreau + (cherry picked from commit 2cefb36087f240b66b2aa4824a317ef5f9b85e68) + Signed-off-by: Christopher Faulet + +diff --git a/src/checks.c b/src/checks.c +index e04f1146..fdebf931 100644 +--- a/src/checks.c ++++ b/src/checks.c +@@ -2771,7 +2771,7 @@ static int tcpcheck_main(struct check *check) + conn_install_mux(conn, &mux_pt_ops, cs); + + ret = SF_ERR_INTERNAL; +- if (proto->connect) ++ if (proto && proto->connect) + ret = proto->connect(conn, + 1 /* I/O polling is always needed */, + (next && next->action == TCPCHK_ACT_EXPECT) ? 0 : 2); diff --git a/net/haproxy/patches/013-BUILD-makefile-work-around-an-old-bug-in-GNU-make-3-80.patch b/net/haproxy/patches/013-BUILD-makefile-work-around-an-old-bug-in-GNU-make-3-80.patch new file mode 100644 index 0000000000..ba4ec2c043 --- /dev/null +++ b/net/haproxy/patches/013-BUILD-makefile-work-around-an-old-bug-in-GNU-make-3-80.patch @@ -0,0 +1,35 @@ +commit 795773be8c3ddc8380f134adc7e2ccfde2d8469b +Author: Willy Tarreau +Date: Fri Mar 29 17:17:52 2019 +0100 + + BUILD: makefile: work around an old bug in GNU make-3.80 + + GNU make-3.80 fails on the .build_opts target, expecting the closing + brace before the first semi-colon in the shell command, it probably + uses a more limited parser for dependencies. Actually it appears it's + enough to place this command in a variable and reference the variable + there. Since it doesn't affect later versions (and the resulting string + is always empty anyway), let's apply the minor change to continue to + comply with the announced dependencies. + + This could be backported as far as 1.6. + + (cherry picked from commit 509a009c5dd06e680bc2fff6ebc45f7f42aaee3e) + Signed-off-by: Willy Tarreau + (cherry picked from commit 953b732eef96689f2b11bc2768ba05f28feac9a5) + Signed-off-by: Christopher Faulet + +diff --git a/Makefile b/Makefile +index 94e04738..bdf5a2d0 100644 +--- a/Makefile ++++ b/Makefile +@@ -905,7 +905,8 @@ INCLUDES = $(wildcard include/*/*.h ebtree/*.h) + DEP = $(INCLUDES) .build_opts + + # Used only to force a rebuild if some build options change +-.build_opts: $(shell rm -f .build_opts.new; echo \'$(TARGET) $(BUILD_OPTIONS) $(VERBOSE_CFLAGS)\' > .build_opts.new; if cmp -s .build_opts .build_opts.new; then rm -f .build_opts.new; else mv -f .build_opts.new .build_opts; fi) ++build_opts = $(shell rm -f .build_opts.new; echo \'$(TARGET) $(BUILD_OPTIONS) $(VERBOSE_CFLAGS)\' > .build_opts.new; if cmp -s .build_opts .build_opts.new; then rm -f .build_opts.new; else mv -f .build_opts.new .build_opts; fi) ++.build_opts: $(build_opts) + + haproxy: $(OPTIONS_OBJS) $(EBTREE_OBJS) $(OBJS) + $(LD) $(LDFLAGS) -o $@ $^ $(LDOPTS) diff --git a/net/haproxy/patches/014-MINOR-tools-make-memvprintf-never-pass-a-NULL-target-to-vsnprintf.patch b/net/haproxy/patches/014-MINOR-tools-make-memvprintf-never-pass-a-NULL-target-to-vsnprintf.patch new file mode 100644 index 0000000000..776fae4b19 --- /dev/null +++ b/net/haproxy/patches/014-MINOR-tools-make-memvprintf-never-pass-a-NULL-target-to-vsnprintf.patch @@ -0,0 +1,42 @@ +commit 36b50ee589a4fda66d222af7de8ad866d30613c7 +Author: Willy Tarreau +Date: Fri Mar 29 19:13:23 2019 +0100 + + MINOR: tools: make memvprintf() never pass a NULL target to vsnprintf() + + Most modern platforms don't touch the output buffer when the size + argument is null, but there exist a few old ones (like AIX 5 and + possibly Tru64) where the output will be dereferenced anyway, probably + to write the trailing null, crashing the process. memprintf() uses this + to measure the desired length. + + There is a very simple workaround to this consisting in passing a pointer + to a character instead of a NULL pointer. It was confirmed to fix the issue + on AIX 5.1. + + (cherry picked from commit e0609f5f49f55b122e7da9bd1d3b1b786366e80c) + [wt: it likely makes sense to backport this to all supported branches] + Signed-off-by: Willy Tarreau + (cherry picked from commit 1719b6be375bf9478c2cfe78caccd818d37a4d50) + Signed-off-by: Christopher Faulet + +diff --git a/src/standard.c b/src/standard.c +index 38c4f3f2..69cddcea 100644 +--- a/src/standard.c ++++ b/src/standard.c +@@ -3402,12 +3402,14 @@ char *memvprintf(char **out, const char *format, va_list orig_args) + return NULL; + + do { ++ char buf1; ++ + /* vsnprintf() will return the required length even when the + * target buffer is NULL. We do this in a loop just in case + * intermediate evaluations get wrong. + */ + va_copy(args, orig_args); +- needed = vsnprintf(ret, allocated, format, args); ++ needed = vsnprintf(ret ? ret : &buf1, allocated, format, args); + va_end(args); + if (needed < allocated) { + /* Note: on Solaris 8, the first iteration always diff --git a/net/haproxy/patches/015-BUILD-makefile-fix-build-of-IPv6-header-on-aix51.patch b/net/haproxy/patches/015-BUILD-makefile-fix-build-of-IPv6-header-on-aix51.patch new file mode 100644 index 0000000000..6a81dc4a64 --- /dev/null +++ b/net/haproxy/patches/015-BUILD-makefile-fix-build-of-IPv6-header-on-aix51.patch @@ -0,0 +1,27 @@ +commit 369d595d95f4945f0e14dd02757cf64eaf8512d9 +Author: Willy Tarreau +Date: Fri Mar 29 17:40:23 2019 +0100 + + BUILD: makefile: fix build of IPv6 header on aix51 + + ip6_hdr is called ip6hdr there and is only defined when STEVENS_API is + defined. + + (cherry picked from commit 6f4fd1b183fc70863926bdc31d7f0b4739c92c56) + Signed-off-by: Willy Tarreau + (cherry picked from commit 99903a0c6419edb515a07a6e1fba4b8bc78babd1) + Signed-off-by: Christopher Faulet + +diff --git a/Makefile b/Makefile +index bdf5a2d0..81e70e02 100644 +--- a/Makefile ++++ b/Makefile +@@ -358,7 +358,7 @@ ifeq ($(TARGET),aix51) + # This is for AIX 5.1 + USE_POLL = implicit + USE_LIBCRYPT = implicit +- TARGET_CFLAGS = -Dss_family=__ss_family ++ TARGET_CFLAGS = -Dss_family=__ss_family -Dip6_hdr=ip6hdr -DSTEVENS_API + DEBUG_CFLAGS = + else + ifeq ($(TARGET),aix52) diff --git a/net/haproxy/patches/016-BUILD-makefile-add-_LINUX_SOURCE_COMPAT-to-build-on-AIX-51.patch b/net/haproxy/patches/016-BUILD-makefile-add-_LINUX_SOURCE_COMPAT-to-build-on-AIX-51.patch new file mode 100644 index 0000000000..9c7d9809cf --- /dev/null +++ b/net/haproxy/patches/016-BUILD-makefile-add-_LINUX_SOURCE_COMPAT-to-build-on-AIX-51.patch @@ -0,0 +1,27 @@ +commit 0ddf04ee66d3a2ff9c47d0c91a2d3c7bc06b34e9 +Author: Willy Tarreau +Date: Fri Mar 29 17:56:13 2019 +0100 + + BUILD: makefile: add _LINUX_SOURCE_COMPAT to build on AIX-51 + + Not tested on later versions, but at least there _LINUX_SOURCE_COMPAT + must be defined to access the CMSG_SPACE() and CMSG_LEN() macros. + + (cherry picked from commit 891d65a67280bea35f8ae38e4dcfe7c2330ddd10) + Signed-off-by: Willy Tarreau + (cherry picked from commit 20faac416072cabffb13a7e41a58dd5b56fc21b0) + Signed-off-by: Christopher Faulet + +diff --git a/Makefile b/Makefile +index 81e70e02..955f1188 100644 +--- a/Makefile ++++ b/Makefile +@@ -358,7 +358,7 @@ ifeq ($(TARGET),aix51) + # This is for AIX 5.1 + USE_POLL = implicit + USE_LIBCRYPT = implicit +- TARGET_CFLAGS = -Dss_family=__ss_family -Dip6_hdr=ip6hdr -DSTEVENS_API ++ TARGET_CFLAGS = -Dss_family=__ss_family -Dip6_hdr=ip6hdr -DSTEVENS_API -D_LINUX_SOURCE_COMPAT + DEBUG_CFLAGS = + else + ifeq ($(TARGET),aix52) diff --git a/net/haproxy/patches/017-BUILD-Makefile-disable-shared-cache-on-AIX-5-1.patch b/net/haproxy/patches/017-BUILD-Makefile-disable-shared-cache-on-AIX-5-1.patch new file mode 100644 index 0000000000..39fe46d515 --- /dev/null +++ b/net/haproxy/patches/017-BUILD-Makefile-disable-shared-cache-on-AIX-5-1.patch @@ -0,0 +1,35 @@ +commit f78f501857cec8bb682b76faa9cbd5cc1e0dcad4 +Author: Willy Tarreau +Date: Fri Mar 29 18:56:54 2019 +0100 + + BUILD: Makefile: disable shared cache on AIX 5.1 + + AIX 5.1 is missing the following builtins used for atomic locking of the + shared inter-process cache : + + .__sync_val_compare_and_swap_4 + .__sync_lock_test_and_set_4 + .__sync_sub_and_fetch_4 + + Let's simply use the private cache by default since nobody cares on + such old systems. No test was made on a more recent version. + + (cherry picked from commit 13d9b0231abe6e1d6c404725746308a98bdab8c6) + Signed-off-by: Willy Tarreau + (cherry picked from commit 3349375d3f5820f261f9c3058ca6e5ced67f3505) + Signed-off-by: Christopher Faulet + +diff --git a/Makefile b/Makefile +index 955f1188..36df7b29 100644 +--- a/Makefile ++++ b/Makefile +@@ -358,7 +358,8 @@ ifeq ($(TARGET),aix51) + # This is for AIX 5.1 + USE_POLL = implicit + USE_LIBCRYPT = implicit +- TARGET_CFLAGS = -Dss_family=__ss_family -Dip6_hdr=ip6hdr -DSTEVENS_API -D_LINUX_SOURCE_COMPAT ++ USE_PRIVATE_CACHE = implicit ++ TARGET_CFLAGS = -Dss_family=__ss_family -Dip6_hdr=ip6hdr -DSTEVENS_API -D_LINUX_SOURCE_COMPAT -Dunsetenv=my_unsetenv + DEBUG_CFLAGS = + else + ifeq ($(TARGET),aix52) diff --git a/net/haproxy/patches/018-BUG-MINOR-cli-correctly-handle-abns-in-show-cli-sockets.patch b/net/haproxy/patches/018-BUG-MINOR-cli-correctly-handle-abns-in-show-cli-sockets.patch new file mode 100644 index 0000000000..baf41ee782 --- /dev/null +++ b/net/haproxy/patches/018-BUG-MINOR-cli-correctly-handle-abns-in-show-cli-sockets.patch @@ -0,0 +1,33 @@ +commit 9ffd35ec59b1ade094b4828b880bdb4971f0c69e +Author: William Lallemand +Date: Mon Apr 1 11:30:04 2019 +0200 + + BUG/MINOR: cli: correctly handle abns in 'show cli sockets' + + The 'show cli sockets' was not handling the abns sockets. This is a + problem since it uses the AF_UNIX family, it displays nothing + in the path column because the path starts by \0. + + Should be backported to 1.9 and 1.8. + + (cherry picked from commit 75812a7a3cdc853ceee0b3fb2173db060057ba35) + Signed-off-by: Christopher Faulet + (cherry picked from commit d06619b5adf8a718ad840dbddbc9d2c83fbb01b1) + Signed-off-by: Christopher Faulet + +diff --git a/src/cli.c b/src/cli.c +index 079d8d9b..39744c7a 100644 +--- a/src/cli.c ++++ b/src/cli.c +@@ -972,7 +972,10 @@ static int cli_io_handler_show_cli_sock(struct appctx *appctx) + const struct sockaddr_un *un; + + un = (struct sockaddr_un *)&l->addr; +- chunk_appendf(&trash, "%s ", un->sun_path); ++ if (un->sun_path[0] == '\0') ++ chunk_appendf(&trash, "abns@%s ", un->sun_path+1); ++ else ++ chunk_appendf(&trash, "%s ", un->sun_path); + } else if (l->addr.ss_family == AF_INET) { + addr_to_str(&l->addr, addr, sizeof(addr)); + port_to_str(&l->addr, port, sizeof(port)); diff --git a/net/haproxy/patches/019-MINOR-cli-start-addresses-by-a-prefix-in-show-cli-sockets.patch b/net/haproxy/patches/019-MINOR-cli-start-addresses-by-a-prefix-in-show-cli-sockets.patch new file mode 100644 index 0000000000..006305cb6d --- /dev/null +++ b/net/haproxy/patches/019-MINOR-cli-start-addresses-by-a-prefix-in-show-cli-sockets.patch @@ -0,0 +1,45 @@ +commit d33b1dd5193df4dfc1dcbea22ade1ee53c89009e +Author: William Lallemand +Date: Mon Apr 1 11:30:05 2019 +0200 + + MINOR: cli: start addresses by a prefix in 'show cli sockets' + + Displays a prefix for every addresses in 'show cli sockets'. + It could be 'unix@', 'ipv4@', 'ipv6@', 'abns@' or 'sockpair@'. + + Could be backported in 1.9 and 1.8. + + (cherry picked from commit e58915f07f7e7a83a7aece909f115bb40887cd55) + Signed-off-by: Christopher Faulet + (cherry picked from commit 97a9b60016159140ba3c001f7427d8ff52d1d311) + Signed-off-by: Christopher Faulet + +diff --git a/src/cli.c b/src/cli.c +index 39744c7a..f0c41b06 100644 +--- a/src/cli.c ++++ b/src/cli.c +@@ -972,18 +972,19 @@ static int cli_io_handler_show_cli_sock(struct appctx *appctx) + const struct sockaddr_un *un; + + un = (struct sockaddr_un *)&l->addr; +- if (un->sun_path[0] == '\0') ++ if (un->sun_path[0] == '\0') { + chunk_appendf(&trash, "abns@%s ", un->sun_path+1); +- else +- chunk_appendf(&trash, "%s ", un->sun_path); ++ } else { ++ chunk_appendf(&trash, "unix@%s ", un->sun_path); ++ } + } else if (l->addr.ss_family == AF_INET) { + addr_to_str(&l->addr, addr, sizeof(addr)); + port_to_str(&l->addr, port, sizeof(port)); +- chunk_appendf(&trash, "%s:%s ", addr, port); ++ chunk_appendf(&trash, "ipv4@%s:%s ", addr, port); + } else if (l->addr.ss_family == AF_INET6) { + addr_to_str(&l->addr, addr, sizeof(addr)); + port_to_str(&l->addr, port, sizeof(port)); +- chunk_appendf(&trash, "[%s]:%s ", addr, port); ++ chunk_appendf(&trash, "ipv6@[%s]:%s ", addr, port); + } else + continue; + diff --git a/net/haproxy/patches/020-BUG-MEDIUM-peers-fix-a-case-where-peer-session-is-not-cleanly-reset-on-release.patch b/net/haproxy/patches/020-BUG-MEDIUM-peers-fix-a-case-where-peer-session-is-not-cleanly-reset-on-release.patch new file mode 100644 index 0000000000..277667dc10 --- /dev/null +++ b/net/haproxy/patches/020-BUG-MEDIUM-peers-fix-a-case-where-peer-session-is-not-cleanly-reset-on-release.patch @@ -0,0 +1,168 @@ +commit 3bb33335816c1c9549d21bcc14bed29519b938a3 +Author: Emeric Brun +Date: Tue Apr 2 17:22:01 2019 +0200 + + BUG/MEDIUM: peers: fix a case where peer session is not cleanly reset on release. + + The deinit took place in only peer_session_release, but in the a case of a + previous call to peer_session_forceshutdown, the session cursors + won't be reset, resulting in a bad state for new session of the same + peer. For instance, a table definition message could be dropped and + so all update messages will be dropped by the remote peer. + + This patch move the deinit processing directly in the force shutdown + funtion. Killed session remains in "ST_END" state but ref on peer was + reset to NULL and deinit will be skipped on session release function. + + The session release continue to assure the deinit for "active" sessions. + + This patch should be backported on all stable version since proto + peers v2. + + (cherry picked from commit 9ef2ad7844e577b505019695c59284f4a439fc33) + Signed-off-by: Christopher Faulet + (cherry picked from commit 14831989a081f3944cf891afd56e6d9f6086c3ed) + [cf: global variabled connected_peers and active_peers don't exist in 1.8] + Signed-off-by: Christopher Faulet + +diff --git a/src/peers.c b/src/peers.c +index 465ffe85..d7dc51d8 100644 +--- a/src/peers.c ++++ b/src/peers.c +@@ -172,7 +172,7 @@ enum { + #define PEER_DWNGRD_MINOR_VER 0 + + struct peers *cfg_peers = NULL; +-static void peer_session_forceshutdown(struct appctx *appctx); ++static void peer_session_forceshutdown(struct peer *peer); + + /* This function encode an uint64 to 'dynamic' length format. + The encoded value is written at address *str, and the +@@ -492,15 +492,53 @@ static int peer_prepare_ackmsg(struct shared_table *st, char *msg, size_t size) + return (cursor - msg) + datalen; + } + ++/* ++ * Function to deinit connected peer ++ */ ++void __peer_session_deinit(struct peer *peer) ++{ ++ struct stream_interface *si; ++ struct stream *s; ++ struct peers *peers; ++ ++ if (!peer->appctx) ++ return; ++ ++ si = peer->appctx->owner; ++ if (!si) ++ return; ++ ++ s = si_strm(si); ++ if (!s) ++ return; ++ ++ peers = strm_fe(s)->parent; ++ if (!peers) ++ return; ++ ++ /* Re-init current table pointers to force announcement on re-connect */ ++ peer->remote_table = peer->last_local_table = NULL; ++ peer->appctx = NULL; ++ if (peer->flags & PEER_F_LEARN_ASSIGN) { ++ /* unassign current peer for learning */ ++ peer->flags &= ~(PEER_F_LEARN_ASSIGN); ++ peers->flags &= ~(PEERS_F_RESYNC_ASSIGN|PEERS_F_RESYNC_PROCESS); ++ ++ /* reschedule a resync */ ++ peers->resync_timeout = tick_add(now_ms, MS_TO_TICKS(5000)); ++ } ++ /* reset teaching and learning flags to 0 */ ++ peer->flags &= PEER_TEACH_RESET; ++ peer->flags &= PEER_LEARN_RESET; ++ task_wakeup(peers->sync_task, TASK_WOKEN_MSG); ++} ++ + /* + * Callback to release a session with a peer + */ + static void peer_session_release(struct appctx *appctx) + { +- struct stream_interface *si = appctx->owner; +- struct stream *s = si_strm(si); + struct peer *peer = appctx->ctx.peers.ptr; +- struct peers *peers = strm_fe(s)->parent; + + /* appctx->ctx.peers.ptr is not a peer session */ + if (appctx->st0 < PEER_SESS_ST_SENDSUCCESS) +@@ -509,24 +547,9 @@ static void peer_session_release(struct appctx *appctx) + /* peer session identified */ + if (peer) { + HA_SPIN_LOCK(PEER_LOCK, &peer->lock); +- if (peer->appctx == appctx) { +- /* Re-init current table pointers to force announcement on re-connect */ +- peer->remote_table = peer->last_local_table = NULL; +- peer->appctx = NULL; +- if (peer->flags & PEER_F_LEARN_ASSIGN) { +- /* unassign current peer for learning */ +- peer->flags &= ~(PEER_F_LEARN_ASSIGN); +- peers->flags &= ~(PEERS_F_RESYNC_ASSIGN|PEERS_F_RESYNC_PROCESS); +- +- /* reschedule a resync */ +- peers->resync_timeout = tick_add(now_ms, MS_TO_TICKS(5000)); +- } +- /* reset teaching and learning flags to 0 */ +- peer->flags &= PEER_TEACH_RESET; +- peer->flags &= PEER_LEARN_RESET; +- } ++ if (peer->appctx == appctx) ++ __peer_session_deinit(peer); + HA_SPIN_UNLOCK(PEER_LOCK, &peer->lock); +- task_wakeup(peers->sync_task, TASK_WOKEN_MSG); + } + } + +@@ -704,7 +727,7 @@ switchstate: + * for a while. + */ + curpeer->reconnect = tick_add(now_ms, MS_TO_TICKS(50 + random() % 2000)); +- peer_session_forceshutdown(curpeer->appctx); ++ peer_session_forceshutdown(curpeer); + } + if (maj_ver != (unsigned int)-1 && min_ver != (unsigned int)-1) { + if (min_ver == PEER_DWNGRD_MINOR_VER) { +@@ -1832,11 +1855,14 @@ static struct applet peer_applet = { + .release = peer_session_release, + }; + ++ + /* + * Use this function to force a close of a peer session + */ +-static void peer_session_forceshutdown(struct appctx *appctx) ++static void peer_session_forceshutdown(struct peer *peer) + { ++ struct appctx *appctx = peer->appctx; ++ + /* Note that the peer sessions which have just been created + * (->st0 == PEER_SESS_ST_CONNECT) must not + * be shutdown, if not, the TCP session will never be closed +@@ -1849,6 +1875,8 @@ static void peer_session_forceshutdown(struct appctx *appctx) + if (appctx->applet != &peer_applet) + return; + ++ __peer_session_deinit(peer); ++ + appctx->st0 = PEER_SESS_ST_END; + appctx_wakeup(appctx); + } +@@ -2094,8 +2122,7 @@ static struct task *process_peer_sync(struct task * task) + */ + ps->reconnect = tick_add(now_ms, MS_TO_TICKS(50 + random() % 2000)); + if (ps->appctx) { +- peer_session_forceshutdown(ps->appctx); +- ps->appctx = NULL; ++ peer_session_forceshutdown(ps); + } + } + } diff --git a/net/haproxy/patches/021-BUILD-use-inttypes-h-instead-of-stdint-h.patch b/net/haproxy/patches/021-BUILD-use-inttypes-h-instead-of-stdint-h.patch new file mode 100644 index 0000000000..ca79b057c9 --- /dev/null +++ b/net/haproxy/patches/021-BUILD-use-inttypes-h-instead-of-stdint-h.patch @@ -0,0 +1,251 @@ +commit 7948348cdc115389700242901c91d323192850a8 +Author: Willy Tarreau +Date: Fri Mar 29 17:26:33 2019 +0100 + + BUILD: use inttypes.h instead of stdint.h + + I found on an (old) AIX 5.1 machine that stdint.h didn't exist while + inttypes.h which is expected to include it does exist and provides the + desired functionalities. + + As explained here, stdint being just a subset of inttypes for use in + freestanding environments, it's probably always OK to switch to inttypes + instead: + + https://pubs.opengroup.org/onlinepubs/009696799/basedefs/stdint.h.html + + Also it's even clearer here in the autoconf doc : + + https://www.gnu.org/software/autoconf/manual/autoconf-2.61/html_node/Header-Portability.html + + "The C99 standard says that inttypes.h includes stdint.h, so there's + no need to include stdint.h separately in a standard environment. + Some implementations have inttypes.h but not stdint.h (e.g., Solaris + 7), but we don't know of any implementation that has stdint.h but not + inttypes.h" + + (cherry picked from commit a1bd1faeebd03825677d111a1350ee04d625f165) + [wt: dropped include/proto/protocol_buffers.h] + Signed-off-by: Willy Tarreau + (cherry picked from commit 19f364bb4ad0205e134dc527e5164b7a21c2ad07) + [cf: missing files removed] + Signed-off-by: Christopher Faulet + +diff --git a/contrib/hpack/gen-rht.c b/contrib/hpack/gen-rht.c +index b1b90317..4260ffbe 100644 +--- a/contrib/hpack/gen-rht.c ++++ b/contrib/hpack/gen-rht.c +@@ -9,7 +9,7 @@ + * 00 => 0x0a, 01 => 0x0d, 10 => 0x16, 11 => EOS + */ + +-#include ++#include + #include + #include + #include +diff --git a/contrib/plug_qdisc/plug_qdisc.c b/contrib/plug_qdisc/plug_qdisc.c +index 294994eb..606a834c 100644 +--- a/contrib/plug_qdisc/plug_qdisc.c ++++ b/contrib/plug_qdisc/plug_qdisc.c +@@ -1,4 +1,4 @@ +-#include ++#include + #include + #include + #include +diff --git a/contrib/spoa_example/include/spop_functions.h b/contrib/spoa_example/include/spop_functions.h +index 8319e41b..cff45321 100644 +--- a/contrib/spoa_example/include/spop_functions.h ++++ b/contrib/spoa_example/include/spop_functions.h +@@ -1,7 +1,7 @@ + #ifndef _SPOP_FUNCTIONS_H + #define _SPOP_FUNCTIONS_H + +-#include ++#include + #include + #include + +diff --git a/contrib/wireshark-dissectors/peers/packet-happp.c b/contrib/wireshark-dissectors/peers/packet-happp.c +index a1983316..6fca353a 100644 +--- a/contrib/wireshark-dissectors/peers/packet-happp.c ++++ b/contrib/wireshark-dissectors/peers/packet-happp.c +@@ -22,7 +22,7 @@ + */ + + #include +-#include ++#include + #include + #include + +diff --git a/include/common/hpack-dec.h b/include/common/hpack-dec.h +index b03398a4..bea2d521 100644 +--- a/include/common/hpack-dec.h ++++ b/include/common/hpack-dec.h +@@ -28,7 +28,7 @@ + #ifndef _COMMON_HPACK_DEC_H + #define _COMMON_HPACK_DEC_H + +-#include ++#include + #include + #include + #include +diff --git a/include/common/hpack-enc.h b/include/common/hpack-enc.h +index 0a44dfc7..fee56fd4 100644 +--- a/include/common/hpack-enc.h ++++ b/include/common/hpack-enc.h +@@ -28,7 +28,7 @@ + #ifndef _COMMON_HPACK_ENC_H + #define _COMMON_HPACK_ENC_H + +-#include ++#include + #include + #include + #include +diff --git a/include/common/hpack-huff.h b/include/common/hpack-huff.h +index 85ca4171..04276d2c 100644 +--- a/include/common/hpack-huff.h ++++ b/include/common/hpack-huff.h +@@ -27,7 +27,7 @@ + #ifndef _PROTO_HPACK_HUFF_H + #define _PROTO_HPACK_HUFF_H + +-#include ++#include + + int huff_enc(const char *s, char *out); + int huff_dec(const uint8_t *huff, int hlen, char *out, int olen); +diff --git a/include/common/hpack-tbl.h b/include/common/hpack-tbl.h +index 2cbc2bf6..ca3f2aa9 100644 +--- a/include/common/hpack-tbl.h ++++ b/include/common/hpack-tbl.h +@@ -27,7 +27,7 @@ + #ifndef _COMMON_HPACK_TBL_H + #define _COMMON_HPACK_TBL_H + +-#include ++#include + #include + #include + #include +diff --git a/include/common/http-hdr.h b/include/common/http-hdr.h +index a0bb341b..db1bfa14 100644 +--- a/include/common/http-hdr.h ++++ b/include/common/http-hdr.h +@@ -27,7 +27,7 @@ + #ifndef _COMMON_HTTP_HDR_H + #define _COMMON_HTTP_HDR_H + +-#include ++#include + #include + + /* a header field made of a name and a value. Such structure stores 4 longs so +diff --git a/include/proto/shctx.h b/include/proto/shctx.h +index 55cb2a77..3b42ed3b 100644 +--- a/include/proto/shctx.h ++++ b/include/proto/shctx.h +@@ -17,7 +17,7 @@ + #include + #include + +-#include ++#include + + #ifndef USE_PRIVATE_CACHE + #ifdef USE_PTHREAD_PSHARED +diff --git a/src/h2.c b/src/h2.c +index 5c83d6b6..d7a03405 100644 +--- a/src/h2.c ++++ b/src/h2.c +@@ -25,7 +25,7 @@ + * OTHER DEALINGS IN THE SOFTWARE. + */ + +-#include ++#include + #include + #include + #include +diff --git a/src/hpack-dec.c b/src/hpack-dec.c +index 3fd4224d..ad5b23a8 100644 +--- a/src/hpack-dec.c ++++ b/src/hpack-dec.c +@@ -25,7 +25,7 @@ + * OTHER DEALINGS IN THE SOFTWARE. + */ + +-#include ++#include + #include + #include + #include +diff --git a/src/hpack-enc.c b/src/hpack-enc.c +index cb0767ed..685c9f3d 100644 +--- a/src/hpack-enc.c ++++ b/src/hpack-enc.c +@@ -25,7 +25,7 @@ + * OTHER DEALINGS IN THE SOFTWARE. + */ + +-#include ++#include + #include + #include + #include +diff --git a/src/hpack-huff.c b/src/hpack-huff.c +index cbf1fa02..bdcff7fe 100644 +--- a/src/hpack-huff.c ++++ b/src/hpack-huff.c +@@ -26,7 +26,7 @@ + */ + + #include +-#include ++#include + #include + + #include +diff --git a/src/hpack-tbl.c b/src/hpack-tbl.c +index e2d4426f..21aa4bc8 100644 +--- a/src/hpack-tbl.c ++++ b/src/hpack-tbl.c +@@ -25,7 +25,7 @@ + * OTHER DEALINGS IN THE SOFTWARE. + */ + +-#include ++#include + #include + #include + #include +diff --git a/src/sha1.c b/src/sha1.c +index 3b562b55..b7c2d709 100644 +--- a/src/sha1.c ++++ b/src/sha1.c +@@ -26,7 +26,7 @@ + + /* this is only to get definitions for memcpy(), ntohl() and htonl() */ + #include +-#include ++#include + #include + + #include +diff --git a/src/xxhash.c b/src/xxhash.c +index 5702e64f..4792f128 100644 +--- a/src/xxhash.c ++++ b/src/xxhash.c +@@ -98,7 +98,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) + // Basic Types + //************************************** + #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +-# include ++# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; diff --git a/net/haproxy/patches/022-BUILD-connection-fix-naming-of-ip_v-field.patch b/net/haproxy/patches/022-BUILD-connection-fix-naming-of-ip_v-field.patch new file mode 100644 index 0000000000..a3a245b1d8 --- /dev/null +++ b/net/haproxy/patches/022-BUILD-connection-fix-naming-of-ip_v-field.patch @@ -0,0 +1,50 @@ +commit a49725beede82687afd6603384f318afe9e60432 +Author: Willy Tarreau +Date: Fri Mar 29 17:35:32 2019 +0100 + + BUILD: connection: fix naming of ip_v field + + AIX defines ip_v as ip_ff.ip_fv in netinet/ip.h using a macro, and + unfortunately we do have a local variable with such a name and which + uses the same header file. Let's rename the variable to ip_ver to fix + this. + + (cherry picked from commit 0ca24aa028159874d77677076a835930de79ba8d) + Signed-off-by: Willy Tarreau + (cherry picked from commit 0a5e7a9a39e14dccba4caa7df20cd3970f354078) + Signed-off-by: Christopher Faulet + +diff --git a/src/connection.c b/src/connection.c +index 7403e8ae..f57ef60a 100644 +--- a/src/connection.c ++++ b/src/connection.c +@@ -699,7 +699,7 @@ int conn_recv_netscaler_cip(struct connection *conn, int flag) + { + char *line; + uint32_t hdr_len; +- uint8_t ip_v; ++ uint8_t ip_ver; + + /* we might have been called just after an asynchronous shutr */ + if (conn->flags & CO_FL_SOCK_RD_SH) +@@ -765,9 +765,9 @@ int conn_recv_netscaler_cip(struct connection *conn, int flag) + goto missing; + + /* Get IP version from the first four bits */ +- ip_v = (*line & 0xf0) >> 4; ++ ip_ver = (*line & 0xf0) >> 4; + +- if (ip_v == 4) { ++ if (ip_ver == 4) { + struct ip *hdr_ip4; + struct my_tcphdr *hdr_tcp; + +@@ -797,7 +797,7 @@ int conn_recv_netscaler_cip(struct connection *conn, int flag) + + conn->flags |= CO_FL_ADDR_FROM_SET | CO_FL_ADDR_TO_SET; + } +- else if (ip_v == 6) { ++ else if (ip_ver == 6) { + struct ip6_hdr *hdr_ip6; + struct my_tcphdr *hdr_tcp; + diff --git a/net/haproxy/patches/023-BUG-MEDIUM-pattern-assign-pattern-IDs-after-checking-the-config-validity.patch b/net/haproxy/patches/023-BUG-MEDIUM-pattern-assign-pattern-IDs-after-checking-the-config-validity.patch new file mode 100644 index 0000000000..cea387229d --- /dev/null +++ b/net/haproxy/patches/023-BUG-MEDIUM-pattern-assign-pattern-IDs-after-checking-the-config-validity.patch @@ -0,0 +1,54 @@ +commit 3bd00f356783d331deba80de76c989d416e4a52e +Author: Willy Tarreau +Date: Thu Apr 11 14:47:08 2019 +0200 + + BUG/MEDIUM: pattern: assign pattern IDs after checking the config validity + + Pavlos Parissis reported an interesting case where some map identifiers + were not assigned (appearing as -1 in show map). It turns out that it + only happens for log-format expressions parsed in check_config_validity() + that involve maps (log-format, use_backend, unique-id-header), as in the + sample configuration below : + + frontend foo + bind :8001 + unique-id-format %[src,map(addr.lst)] + log-format %[src,map(addr.lst)] + use_backend %[src,map(addr.lst)] + + The reason stems from the initial introduction of unique IDs in 1.5 via + commit af5a29d5f ("MINOR: pattern: Each pattern is identified by unique + id.") : the unique_id assignment was done before calling + check_config_validity() so all maps loaded after this call are not + properly configured. From what the function does, it seems they will not + be able to use a cache, will not have a unique_id assigned and will not + be updatable from the CLI. + + This fix must be backported to all supported versions. + + (cherry picked from commit 0f93672dfea805268d674c97573711fbff7e0e70) + Signed-off-by: William Lallemand + (cherry picked from commit ba475a5b390f58450756da67dbf54bf063f2dbef) + Signed-off-by: Christopher Faulet + +diff --git a/src/haproxy.c b/src/haproxy.c +index 5c3febdd..105cde6f 100644 +--- a/src/haproxy.c ++++ b/src/haproxy.c +@@ -1570,14 +1570,14 @@ static void init(int argc, char **argv) + exit(1); + } + +- pattern_finalize_config(); +- + err_code |= check_config_validity(); + if (err_code & (ERR_ABORT|ERR_FATAL)) { + ha_alert("Fatal errors found in configuration.\n"); + exit(1); + } + ++ pattern_finalize_config(); ++ + /* recompute the amount of per-process memory depending on nbproc and + * the shared SSL cache size (allowed to exist in all processes). + */ diff --git a/net/haproxy/patches/024-BUG-MEDIUM-spoe-Queue-message-only-if-no-SPOE-applet-is-attached-to-the-stream.patch b/net/haproxy/patches/024-BUG-MEDIUM-spoe-Queue-message-only-if-no-SPOE-applet-is-attached-to-the-stream.patch new file mode 100644 index 0000000000..cecf42f0f7 --- /dev/null +++ b/net/haproxy/patches/024-BUG-MEDIUM-spoe-Queue-message-only-if-no-SPOE-applet-is-attached-to-the-stream.patch @@ -0,0 +1,42 @@ +commit 308f39235ca8ce09442bf89cd8aa7f4f6b74f214 +Author: Christopher Faulet +Date: Wed Apr 10 14:02:12 2019 +0200 + + BUG/MEDIUM: spoe: Queue message only if no SPOE applet is attached to the stream + + If a SPOE applet is already attached to a stream to handle its messages, we must + not queue them. Otherwise it could be handled by another applet leading to + errors. This happens with fragmented messages only. When the first framgnent is + sent, the SPOE applet sending it is attached to the stream. It should be used to + send all other fragments. + + This patch must be backported to 1.9 and 1.8. + + (cherry picked from commit 3e86cec05ec9cf848abd8f9a79928410874b778d) + [wla: s/_HA_ATOMIC_ADD/HA_ATOMIC_ADD/ in context] + Signed-off-by: William Lallemand + (cherry picked from commit c3468fe1de262c9977510efb1ae47ff1a04c299c) + Signed-off-by: Christopher Faulet + +diff --git a/src/flt_spoe.c b/src/flt_spoe.c +index 66d26f34..64601e3f 100644 +--- a/src/flt_spoe.c ++++ b/src/flt_spoe.c +@@ -2086,11 +2086,14 @@ spoe_queue_context(struct spoe_context *ctx) + return -1; + } + +- /* Add the SPOE context in the sending queue and update all running +- * info */ +- LIST_ADDQ(&agent->rt[tid].sending_queue, &ctx->list); ++ /* Add the SPOE context in the sending queue if the stream has no applet ++ * already assigned and wakeup all idle applets. Otherwise, don't queue ++ * it. */ + if (agent->rt[tid].sending_rate) + agent->rt[tid].sending_rate--; ++ if (ctx->frag_ctx.spoe_appctx) ++ return 1; ++ LIST_ADDQ(&agent->rt[tid].sending_queue, &ctx->list); + + SPOE_PRINTF(stderr, "%d.%06d [SPOE/%-15s] %s: stream=%p" + " - Add stream in sending queue" diff --git a/net/haproxy/patches/025-BUG-MEDIUM-spoe-Return-an-error-if-nothing-is-encoded-for-fragmented-messages.patch b/net/haproxy/patches/025-BUG-MEDIUM-spoe-Return-an-error-if-nothing-is-encoded-for-fragmented-messages.patch new file mode 100644 index 0000000000..0c89d2dcdb --- /dev/null +++ b/net/haproxy/patches/025-BUG-MEDIUM-spoe-Return-an-error-if-nothing-is-encoded-for-fragmented-messages.patch @@ -0,0 +1,34 @@ +commit ebc65295f5ab943955ea6ae9772932c32e39d02c +Author: Christopher Faulet +Date: Wed Apr 10 14:21:51 2019 +0200 + + BUG/MEDIUM: spoe: Return an error if nothing is encoded for fragmented messages + + If the maximum frame size is very small with a large message or argument name, + it is possible to be unable to encode anything. In such case, it is important to + stop processing returning an error otherwise we will retry in loop to encode the + message, failing each time because of the too small frame size. + + This patch must be backported to 1.9 and 1.8. + + (cherry picked from commit a715ea82eacf4ccf7f447bf4dd4111cc29fe171e) + Signed-off-by: William Lallemand + (cherry picked from commit 3c76e4d79669329ae972f3348e441fea7316813f) + [cf: Adapted to use old buffer API] + Signed-off-by: Christopher Faulet + +diff --git a/src/flt_spoe.c b/src/flt_spoe.c +index 64601e3f..95f30898 100644 +--- a/src/flt_spoe.c ++++ b/src/flt_spoe.c +@@ -2276,7 +2276,9 @@ spoe_encode_messages(struct stream *s, struct spoe_context *ctx, + return 1; + + too_big: +- if (!(agent->flags & SPOE_FL_SND_FRAGMENTATION)) { ++ /* Return an error if fragmentation is unsupported or if nothing has ++ * been encoded because its too big and not splittable. */ ++ if (!(agent->flags & SPOE_FL_SND_FRAGMENTATION) || p == ctx->buffer->p) { + ctx->status_code = SPOE_CTX_ERR_TOO_BIG; + return -1; + } diff --git a/net/haproxy/patches/026-BUG-MINOR-threads-fix-the-process-range-of-thread-masks.patch b/net/haproxy/patches/026-BUG-MINOR-threads-fix-the-process-range-of-thread-masks.patch new file mode 100644 index 0000000000..791cf2c961 --- /dev/null +++ b/net/haproxy/patches/026-BUG-MINOR-threads-fix-the-process-range-of-thread-masks.patch @@ -0,0 +1,30 @@ +commit c56c82b4a55f0e03b28248f1230d6d6cd4f40484 +Author: Willy Tarreau +Date: Sat Feb 2 13:18:01 2019 +0100 + + BUG/MINOR: threads: fix the process range of thread masks + + Commit 421f02e ("MINOR: threads: add a MAX_THREADS define instead of + LONGBITS") used a MAX_THREADS macros to fix threads limits. However, + one change was wrong as it affected the upper bound of the process + loop when setting threads masks. No backport is needed. + + (cherry picked from commit bbcf2b9e0d17d83609ac529a21a258d2f2ea7bb8) + [wt: this should be backported to 1.8 as well] + Signed-off-by: Willy Tarreau + (cherry picked from commit 1a3ae41964f934a4317e37ed0e0680f252dea4af) + Signed-off-by: Christopher Faulet + +diff --git a/src/listener.c b/src/listener.c +index b94d823c..45d9c252 100644 +--- a/src/listener.c ++++ b/src/listener.c +@@ -954,7 +954,7 @@ static int bind_parse_process(char **args, int cur_arg, struct proxy *px, struct + + conf->bind_proc |= proc; + if (thread) { +- for (i = 0; i < MAX_THREADS; i++) ++ for (i = 0; i < LONGBITS; i++) + if (!proc || (proc & (1UL << i))) + conf->bind_thread[i] |= thread; + } diff --git a/net/haproxy/patches/027-MINOR-lists-Implement-locked-variations.patch b/net/haproxy/patches/027-MINOR-lists-Implement-locked-variations.patch new file mode 100644 index 0000000000..b8f082daa6 --- /dev/null +++ b/net/haproxy/patches/027-MINOR-lists-Implement-locked-variations.patch @@ -0,0 +1,175 @@ +commit d66183efc6ee57c86400afba3b9b3f8635d3245e +Author: Olivier Houchard +Date: Fri Jan 18 17:26:26 2019 +0100 + + MINOR: lists: Implement locked variations. + + Implement LIST_ADD_LOCKED(), LIST_ADDQ_LOCKED(), LIST_DEL_LOCKED() and + LIST_POP_LOCKED(). + + LIST_ADD_LOCKED, LIST_ADDQ_LOCKED and LIST_DEL_LOCKED work the same as + LIST_ADD, LIST_ADDQ and LIST_DEL, except before any manipulation it locks + the relevant elements of the list, so it's safe to manipulate the list + with multiple threads. + LIST_POP_LOCKED() removes the first element from the list, and returns its + data. + + (cherry picked from commit a8434ec14612d9a04e50e1c51e58f663dad46e96) + Signed-off-by: Willy Tarreau + (cherry picked from commit 15a9450b508a3c9fc2ad2463931e89157c4331f1) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index 2988d2c2..b3f396ef 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -163,5 +163,149 @@ struct cond_wordlist { + &item->member != (list_head); \ + item = back, back = LIST_ELEM(back->member.n, typeof(back), member)) + ++#include ++#define LLIST_BUSY ((struct list *)1) ++ ++/* ++ * Locked version of list manipulation macros. ++ * It is OK to use those concurrently from multiple threads, as long as the ++ * list is only used with the locked variants. The only "unlocked" macro you ++ * can use with a locked list is LIST_INIT. ++ */ ++#define LIST_ADD_LOCKED(lh, el) \ ++ do { \ ++ while (1) { \ ++ struct list *n; \ ++ struct list *p; \ ++ n = HA_ATOMIC_XCHG(&(lh)->n, LLIST_BUSY); \ ++ if (n == LLIST_BUSY) \ ++ continue; \ ++ __ha_barrier_store(); \ ++ p = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ ++ if (p == LLIST_BUSY) { \ ++ (lh)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ (el)->n = n; \ ++ (el)->p = p; \ ++ n->p = (el); \ ++ __ha_barrier_store(); \ ++ p->n = (el); \ ++ __ha_barrier_store(); \ ++ break; \ ++ } \ ++ } while (0) ++ ++#define LIST_ADDQ_LOCKED(lh, el) \ ++ do { \ ++ while (1) { \ ++ struct list *n; \ ++ struct list *p; \ ++ p = HA_ATOMIC_XCHG(&(lh)->p, LLIST_BUSY); \ ++ if (p == LLIST_BUSY) \ ++ continue; \ ++ __ha_barrier_store(); \ ++ n = HA_ATOMIC_XCHG(&p->n, LLIST_BUSY); \ ++ if (n == LLIST_BUSY) { \ ++ (lh)->n = p; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ (el)->n = n; \ ++ (el)->p = p; \ ++ n->p = (el); \ ++ __ha_barrier_store(); \ ++ p->n = (el); \ ++ __ha_barrier_store(); \ ++ break; \ ++ } \ ++ } while (0) ++ ++#define LIST_DEL_LOCKED(el) \ ++ do { \ ++ while (1) { \ ++ struct list *n, *n2; \ ++ struct list *p, *p2; \ ++ n = HA_ATOMIC_XCHG(&(el)->n, LLIST_BUSY); \ ++ if (n == LLIST_BUSY) \ ++ continue; \ ++ p = HA_ATOMIC_XCHG(&(el)->p, LLIST_BUSY); \ ++ if (p == LLIST_BUSY) { \ ++ (el)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ if (p != (el)) { \ ++ p2 = HA_ATOMIC_XCHG(&p->n, LLIST_BUSY); \ ++ if (p2 == LLIST_BUSY) { \ ++ (el)->p = p; \ ++ (el)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ } \ ++ if (n != (el)) { \ ++ n2 = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ ++ if (n2 == LLIST_BUSY) { \ ++ p2->n = (el); \ ++ (el)->p = p; \ ++ (el)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ } \ ++ n->p = p; \ ++ p->n = n; \ ++ __ha_barrier_store(); \ ++ break; \ ++ } \ ++ } while (0) ++ ++ ++/* Remove the first element from the list, and return it */ ++#define LIST_POP_LOCKED(lh, pt, el) \ ++ ({ \ ++ void *_ret; \ ++ while (1) { \ ++ struct list *n, *n2; \ ++ struct list *p, *p2; \ ++ n = HA_ATOMIC_XCHG(&(lh)->n, LLIST_BUSY); \ ++ if (n == LLIST_BUSY) \ ++ continue; \ ++ if (n == (lh)) { \ ++ (lh)->n = lh; \ ++ _ret = NULL; \ ++ break; \ ++ } \ ++ p = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ ++ if (p == LLIST_BUSY) { \ ++ (lh)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ n2 = HA_ATOMIC_XCHG(&n->n, LLIST_BUSY); \ ++ if (n2 == LLIST_BUSY) { \ ++ n->p = p; \ ++ (lh)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ p2 = HA_ATOMIC_XCHG(&n2->p, LLIST_BUSY); \ ++ if (p2 == LLIST_BUSY) { \ ++ n->n = n2; \ ++ n->p = p; \ ++ (lh)->n = n; \ ++ __ha_barrier_store(); \ ++ continue; \ ++ } \ ++ (lh)->n = n2; \ ++ (n2)->p = (lh); \ ++ __ha_barrier_store(); \ ++ _ret = LIST_ELEM(n, pt, el); \ ++ break; \ ++ } \ ++ (_ret); \ ++ }) + + #endif /* _COMMON_MINI_CLIST_H */ diff --git a/net/haproxy/patches/028-BUG-MEDIUM-lists-Properly-handle-the-case-were-removing-the-first-elt.patch b/net/haproxy/patches/028-BUG-MEDIUM-lists-Properly-handle-the-case-were-removing-the-first-elt.patch new file mode 100644 index 0000000000..80306cea62 --- /dev/null +++ b/net/haproxy/patches/028-BUG-MEDIUM-lists-Properly-handle-the-case-were-removing-the-first-elt.patch @@ -0,0 +1,37 @@ +commit 982bf2a0481648a0885b100cc565add399762028 +Author: Olivier Houchard +Date: Tue Feb 26 18:46:07 2019 +0100 + + BUG/MEDIUM: lists: Properly handle the case we're removing the first elt. + + In LIST_DEL_LOCKED(), initialize p2 to NULL, and only attempt to set it back + to its previous value if we had a previous element, and thus p2 is non-NULL. + + (cherry picked from commit db64489aac2135c6ceceec47fe98dc42c0558466) + Signed-off-by: Willy Tarreau + (cherry picked from commit a3d6a2b9dac9775c55a10ed6b60deca219f06ff6) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index b3f396ef..aebeec38 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -226,7 +226,7 @@ struct cond_wordlist { + do { \ + while (1) { \ + struct list *n, *n2; \ +- struct list *p, *p2; \ ++ struct list *p, *p2 = NULL; \ + n = HA_ATOMIC_XCHG(&(el)->n, LLIST_BUSY); \ + if (n == LLIST_BUSY) \ + continue; \ +@@ -248,7 +248,8 @@ struct cond_wordlist { + if (n != (el)) { \ + n2 = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ + if (n2 == LLIST_BUSY) { \ +- p2->n = (el); \ ++ if (p2 != NULL) \ ++ p2->n = (el); \ + (el)->p = p; \ + (el)->n = n; \ + __ha_barrier_store(); \ diff --git a/net/haproxy/patches/029-BUG-MEDIUM-list-fix-the-rollback-on-addq-in-the-locked-liss.patch b/net/haproxy/patches/029-BUG-MEDIUM-list-fix-the-rollback-on-addq-in-the-locked-liss.patch new file mode 100644 index 0000000000..f959dd88dc --- /dev/null +++ b/net/haproxy/patches/029-BUG-MEDIUM-list-fix-the-rollback-on-addq-in-the-locked-liss.patch @@ -0,0 +1,33 @@ +commit 66322aef6533597ed5100916696c0f552513911d +Author: Willy Tarreau +Date: Thu Feb 28 11:09:56 2019 +0100 + + BUG/MEDIUM: list: fix the rollback on addq in the locked liss + + Commit a8434ec14 ("MINOR: lists: Implement locked variations.") + introduced locked lists which use the elements pointers as locks + for concurrent operations. A copy-paste typo in LIST_ADDQ_LOCKED() + causes corruption in the list in case the next pointer is already + held, as it restores the previous pointer into the next one. It + may impact the server pools. + + This will have to be backported if the commit above is backported. + + (cherry picked from commit bd20ad58748eafbded464f574ed84dbbdad31e8d) + Signed-off-by: Willy Tarreau + (cherry picked from commit d4b726dc9e1659cb7fe3508862dfa8b3d23d823a) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index aebeec38..779f81da 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -208,7 +208,7 @@ struct cond_wordlist { + __ha_barrier_store(); \ + n = HA_ATOMIC_XCHG(&p->n, LLIST_BUSY); \ + if (n == LLIST_BUSY) { \ +- (lh)->n = p; \ ++ (lh)->p = p; \ + __ha_barrier_store(); \ + continue; \ + } \ diff --git a/net/haproxy/patches/030-BUG-MEDIUM-list-fix-LIST_POP_LOCKEDs-removal-of-the-last-pointer.patch b/net/haproxy/patches/030-BUG-MEDIUM-list-fix-LIST_POP_LOCKEDs-removal-of-the-last-pointer.patch new file mode 100644 index 0000000000..92ff5a0ee5 --- /dev/null +++ b/net/haproxy/patches/030-BUG-MEDIUM-list-fix-LIST_POP_LOCKEDs-removal-of-the-last-pointer.patch @@ -0,0 +1,34 @@ +commit 8caa2434cd52ac4056f4ec32b423d11a4dc7eaf5 +Author: Willy Tarreau +Date: Thu Feb 28 15:55:18 2019 +0100 + + BUG/MEDIUM: list: fix LIST_POP_LOCKED's removal of the last pointer + + There was a typo making the last updated pointer be the pre-last element's + prev instead of the last's prev element. It didn't show up during early + tests because the contention is very rare on this one and it's implicitly + recovered when updating the pointers to go to the next element, but it was + clearly visible in the listener_accept() tests by having all threads block + on LIST_POP_LOCKED() with n==p==LLIST_BUSY. + + This will have to be backported if commit a8434ec14 ("MINOR: lists: + Implement locked variations.") is backported. + + (cherry picked from commit 285192564d2a7d6eff077d91758b603ba7f2b10a) + Signed-off-by: Willy Tarreau + (cherry picked from commit 20fcf3a0e188551ef185c02c7fb28314de6e412e) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index 779f81da..51c61051 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -292,7 +292,7 @@ struct cond_wordlist { + __ha_barrier_store(); \ + continue; \ + } \ +- p2 = HA_ATOMIC_XCHG(&n2->p, LLIST_BUSY); \ ++ p2 = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ + if (p2 == LLIST_BUSY) { \ + n->n = n2; \ + n->p = p; \ diff --git a/net/haproxy/patches/031-BUG-MEDIUM-list-add-missing-store-barriers-when-updating-elements-and-head.patch b/net/haproxy/patches/031-BUG-MEDIUM-list-add-missing-store-barriers-when-updating-elements-and-head.patch new file mode 100644 index 0000000000..a4810c7952 --- /dev/null +++ b/net/haproxy/patches/031-BUG-MEDIUM-list-add-missing-store-barriers-when-updating-elements-and-head.patch @@ -0,0 +1,66 @@ +commit cee000f97c9300dbbae50118838c6d3e8d5eb9cf +Author: Willy Tarreau +Date: Thu Feb 28 11:14:22 2019 +0100 + + BUG/MEDIUM: list: add missing store barriers when updating elements and head + + Commit a8434ec14 ("MINOR: lists: Implement locked variations.") + introduced locked lists which use the elements pointers as locks + for concurrent operations. Under heavy stress the lists occasionally + fail. The cause is a missing barrier at some points when updating + the list element and the head : nothing prevents the compiler (or + CPU) from updating the list head first before updating the element, + making another thread jump to a wrong location. This patch simply + adds the missing barriers before these two opeations. + + This will have to be backported if the commit above is backported. + + (cherry picked from commit 690d2ad4d207da5c8821b84ab467090bd515eedf) + Signed-off-by: Willy Tarreau + (cherry picked from commit c2aa5cb8b8ff3af27f1a962541e53f792745bc4a) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index 51c61051..92a995c9 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -189,6 +189,7 @@ struct cond_wordlist { + } \ + (el)->n = n; \ + (el)->p = p; \ ++ __ha_barrier_store(); \ + n->p = (el); \ + __ha_barrier_store(); \ + p->n = (el); \ +@@ -214,6 +215,7 @@ struct cond_wordlist { + } \ + (el)->n = n; \ + (el)->p = p; \ ++ __ha_barrier_store(); \ + n->p = (el); \ + __ha_barrier_store(); \ + p->n = (el); \ +@@ -276,6 +278,7 @@ struct cond_wordlist { + continue; \ + if (n == (lh)) { \ + (lh)->n = lh; \ ++ __ha_barrier_store(); \ + _ret = NULL; \ + break; \ + } \ +@@ -288,6 +291,7 @@ struct cond_wordlist { + n2 = HA_ATOMIC_XCHG(&n->n, LLIST_BUSY); \ + if (n2 == LLIST_BUSY) { \ + n->p = p; \ ++ __ha_barrier_store(); \ + (lh)->n = n; \ + __ha_barrier_store(); \ + continue; \ +@@ -296,6 +300,7 @@ struct cond_wordlist { + if (p2 == LLIST_BUSY) { \ + n->n = n2; \ + n->p = p; \ ++ __ha_barrier_store(); \ + (lh)->n = n; \ + __ha_barrier_store(); \ + continue; \ diff --git a/net/haproxy/patches/032-MINOR-list-make-the-delete-and-pop-operations-idempotent.patch b/net/haproxy/patches/032-MINOR-list-make-the-delete-and-pop-operations-idempotent.patch new file mode 100644 index 0000000000..292b1a4b9b --- /dev/null +++ b/net/haproxy/patches/032-MINOR-list-make-the-delete-and-pop-operations-idempotent.patch @@ -0,0 +1,44 @@ +commit f4be23cefd779894ca85680d85b1d66cd22d42b6 +Author: Willy Tarreau +Date: Thu Feb 28 15:05:53 2019 +0100 + + MINOR: list: make the delete and pop operations idempotent + + These operations previously used to return a "locked" element, which is + a constraint when multiple threads try to delete the same element, because + the second one will block indefinitely. Instead, let's make sure that both + LIST_DEL_LOCKED() and LIST_POP_LOCKED() always reinitialize the element + after deleting it. This ensures that the second thread will immediately + unblock and succeed with the removal. It also secures the pop vs delete + competition that may happen when trying to remove an element that's about + to be dequeued. + + (cherry picked from commit 4c747e86cda5d7eab46390779447f216ce9aa5de) + Signed-off-by: Willy Tarreau + (cherry picked from commit 731853a3e19b2ce314c1626360dc5b1dcba5baa1) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index 92a995c9..d4861ccc 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -261,6 +261,9 @@ struct cond_wordlist { + n->p = p; \ + p->n = n; \ + __ha_barrier_store(); \ ++ (el)->p = (el); \ ++ (el)->n = (el); \ ++ __ha_barrier_store(); \ + break; \ + } \ + } while (0) +@@ -308,6 +311,9 @@ struct cond_wordlist { + (lh)->n = n2; \ + (n2)->p = (lh); \ + __ha_barrier_store(); \ ++ (n)->p = (n); \ ++ (n)->n = (n); \ ++ __ha_barrier_store(); \ + _ret = LIST_ELEM(n, pt, el); \ + break; \ + } \ diff --git a/net/haproxy/patches/033-BUG-MEDIUM-list-correct-fix-for-LIST_POP_LOCKEDs-removal-of-last-element.patch b/net/haproxy/patches/033-BUG-MEDIUM-list-correct-fix-for-LIST_POP_LOCKEDs-removal-of-last-element.patch new file mode 100644 index 0000000000..3a22df987f --- /dev/null +++ b/net/haproxy/patches/033-BUG-MEDIUM-list-correct-fix-for-LIST_POP_LOCKEDs-removal-of-last-element.patch @@ -0,0 +1,44 @@ +commit c4a54648f510771e792ed0c0bc16e4cf8be8bb9a +Author: Willy Tarreau +Date: Thu Feb 28 16:51:28 2019 +0100 + + BUG/MEDIUM: list: correct fix for LIST_POP_LOCKED's removal of last element + + As seen with Olivier, in the end the fix in commit 285192564 ("BUG/MEDIUM: + list: fix LIST_POP_LOCKED's removal of the last pointer") is wrong, + the code there was right but the bug was triggered by another bug in + LIST_ADDQ_LOCKED() which doesn't properly update the list's head by + inserting in the wrong order. + + This will have to be backported if the commit above is backported. + + (cherry picked from commit 4ef6801cd4db450b4ac3a21e58ca5fce5256189b) + Signed-off-by: Willy Tarreau + (cherry picked from commit 92f771c7efd9a82ef189d2be7c2fcfa6a6703e07) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index d4861ccc..9ad53aa0 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -216,9 +216,9 @@ struct cond_wordlist { + (el)->n = n; \ + (el)->p = p; \ + __ha_barrier_store(); \ +- n->p = (el); \ ++ n->n = (el); \ + __ha_barrier_store(); \ +- p->n = (el); \ ++ p->p = (el); \ + __ha_barrier_store(); \ + break; \ + } \ +@@ -299,7 +299,7 @@ struct cond_wordlist { + __ha_barrier_store(); \ + continue; \ + } \ +- p2 = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ ++ p2 = HA_ATOMIC_XCHG(&n2->p, LLIST_BUSY); \ + if (p2 == LLIST_BUSY) { \ + n->n = n2; \ + n->p = p; \ diff --git a/net/haproxy/patches/034-BUG-MEDIUM-list-fix-again-LIST_ADDQ_LOCKED.patch b/net/haproxy/patches/034-BUG-MEDIUM-list-fix-again-LIST_ADDQ_LOCKED.patch new file mode 100644 index 0000000000..5a4770ab0e --- /dev/null +++ b/net/haproxy/patches/034-BUG-MEDIUM-list-fix-again-LIST_ADDQ_LOCKED.patch @@ -0,0 +1,43 @@ +commit 43dbd48e8f39b56f77493a4a0e786bd310a8e682 +Author: Willy Tarreau +Date: Mon Mar 4 11:19:49 2019 +0100 + + BUG/MEDIUM: list: fix again LIST_ADDQ_LOCKED + + Well, that's becoming embarrassing. Now this fixes commit 4ef6801c + ("BUG/MEDIUM: list: correct fix for LIST_POP_LOCKED's removal of last + element") which itself tried to fix commit 285192564. This fix only + works under low contention and was tested with the listener's queue. + With the idle conns it's obvious that it's still wrong since adding + more than one element to the list leaves a LLIST_BUSY pointer into + the list's head. This was visible when accumulating idle connections + in a server's list. + + This new version of the fix almost goes back to the original code, + except that since then we addressed issues with expectedly idempotent + operations that were not. Now the code has been verified on paper again + and has survived 300 million connections spread over 4 threads. + + This will have to be backported if the commit above is backported. + + (cherry picked from commit 967de20a43665715e4513c7cc7a67e36a36a9955) + Signed-off-by: Willy Tarreau + (cherry picked from commit 76eeabc45943cc6a493a2212130cedcde803e432) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index 9ad53aa0..27128a2c 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -216,9 +216,9 @@ struct cond_wordlist { + (el)->n = n; \ + (el)->p = p; \ + __ha_barrier_store(); \ +- n->n = (el); \ ++ p->n = (el); \ + __ha_barrier_store(); \ +- p->p = (el); \ ++ n->p = (el); \ + __ha_barrier_store(); \ + break; \ + } \ diff --git a/net/haproxy/patches/035-BUG-MEDIUM-list-fix-incorrect-pointer-unlocking-in-LIST_DEL_LOCKED.patch b/net/haproxy/patches/035-BUG-MEDIUM-list-fix-incorrect-pointer-unlocking-in-LIST_DEL_LOCKED.patch new file mode 100644 index 0000000000..c1b0d3127f --- /dev/null +++ b/net/haproxy/patches/035-BUG-MEDIUM-list-fix-incorrect-pointer-unlocking-in-LIST_DEL_LOCKED.patch @@ -0,0 +1,32 @@ +commit 4c20fedc4302aec6e5f4409274b832d524aef44d +Author: Willy Tarreau +Date: Wed Mar 13 14:03:28 2019 +0100 + + BUG/MEDIUM: list: fix incorrect pointer unlocking in LIST_DEL_LOCKED() + + Injecting on a saturated listener started to exhibit some deadlocks + again between LIST_POP_LOCKED() and LIST_DEL_LOCKED(). Olivier found + it was due to a leftover from a previous debugging session. This patch + fixes it. + + This will have to be backported if the other LIST_*_LOCKED() patches + are backported. + + (cherry picked from commit b0cef35b097c89ff675e055186e71239e105eb01) + Signed-off-by: Willy Tarreau + (cherry picked from commit bf7cc16958e288219989949add02c8a97ed9cf6f) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/mini-clist.h b/include/common/mini-clist.h +index 27128a2c..454089ff 100644 +--- a/include/common/mini-clist.h ++++ b/include/common/mini-clist.h +@@ -251,7 +251,7 @@ struct cond_wordlist { + n2 = HA_ATOMIC_XCHG(&n->p, LLIST_BUSY); \ + if (n2 == LLIST_BUSY) { \ + if (p2 != NULL) \ +- p2->n = (el); \ ++ p->n = p2; \ + (el)->p = p; \ + (el)->n = n; \ + __ha_barrier_store(); \ diff --git a/net/haproxy/patches/036-MAJOR-listener-do-not-hold-the-listener-lock-in-listener_accept.patch b/net/haproxy/patches/036-MAJOR-listener-do-not-hold-the-listener-lock-in-listener_accept.patch new file mode 100644 index 0000000000..f30696cdc2 --- /dev/null +++ b/net/haproxy/patches/036-MAJOR-listener-do-not-hold-the-listener-lock-in-listener_accept.patch @@ -0,0 +1,274 @@ +commit db4bf546d72a37f998a7bf25f00126611af58184 +Author: Willy Tarreau +Date: Mon Feb 25 19:23:37 2019 +0100 + + MAJOR: listener: do not hold the listener lock in listener_accept() + + This function used to hold the listener's lock as a way to stay safe + against concurrent manipulations, but it turns out this is wrong. First, + the lock is held during l->accept(), which itself might indirectly call + listener_release(), which, if the listener is marked full, could result + in __resume_listener() to be called and the lock being taken twice. In + practice it doesn't happen right now because the listener's FULL state + cannot change while we're doing this. + + Second, all the code does is now protected against concurrent accesses. + It used not to be the case in the early days of threads : the frequency + counters are thread-safe. The rate limiting doesn't require extreme + precision. Only the nbconn check is not thread safe. + + Third, the parts called here will have to be called from different + threads without holding this lock, and this becomes a bigger issue + if we need to keep this one. + + This patch does 3 things which need to be addressed at once : + 1) it moves the lock to the only 2 functions that were not protected + since called form listener_accept() : + - limit_listener() + - listener_full() + + 2) it makes sure delete_listener() properly checks its state within + the lock. + + 3) it updates the l->nbconn tracking to make sure that it is always + properly reported and accounted for. There is a point of particular + care around the situation where the listener's maxconn is reached + because the listener has to be marked full before accepting the + connection, then resumed if the connection finally gets dropped. + It is not possible to perform this change without removing the + lock due to the deadlock issue explained above. + + This patch almost doubles the accept rate in multi-thread on a shared + port between 8 threads, and multiplies by 4 the connection rate on a + tcp-request connection reject rule. + + (cherry picked from commit 3f0d02bbc2d12cd083fe1c1a051c42cdccb7bc4a) + [wt: this patch is mandatory for the next fixes to be backported, + it has been cooking almost 2 months in 2.0 with no bad behaviour + and seems reasonable to backport now] + + Signed-off-by: Willy Tarreau + (cherry picked from commit b44ae2d0a6eeffe2af3edd37f7829a8d26b85d43) + Signed-off-by: Christopher Faulet + +diff --git a/src/listener.c b/src/listener.c +index 45d9c252..9ef3d5e8 100644 +--- a/src/listener.c ++++ b/src/listener.c +@@ -226,32 +226,30 @@ int resume_listener(struct listener *l) + + /* Marks a ready listener as full so that the stream code tries to re-enable + * it upon next close() using resume_listener(). +- * +- * Note: this function is only called from listener_accept so is already +- * locked. + */ + static void listener_full(struct listener *l) + { ++ HA_SPIN_LOCK(LISTENER_LOCK, &l->lock); + if (l->state >= LI_READY) { + if (l->state == LI_LIMITED) { + HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); + LIST_DEL(&l->wait_queue); + HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); + } +- +- fd_stop_recv(l->fd); +- l->state = LI_FULL; ++ if (l->state != LI_FULL) { ++ fd_stop_recv(l->fd); ++ l->state = LI_FULL; ++ } + } ++ HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock); + } + + /* Marks a ready listener as limited so that we only try to re-enable it when + * resources are free again. It will be queued into the specified queue. +- * +- * Note: this function is only called from listener_accept so is already +- * locked. + */ + static void limit_listener(struct listener *l, struct list *list) + { ++ HA_SPIN_LOCK(LISTENER_LOCK, &l->lock); + if (l->state == LI_READY) { + HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); + LIST_ADDQ(list, &l->wait_queue); +@@ -259,6 +257,7 @@ static void limit_listener(struct listener *l, struct list *list) + fd_stop_recv(l->fd); + l->state = LI_LIMITED; + } ++ HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock); + } + + /* This function adds all of the protocol's listener's file descriptors to the +@@ -422,15 +421,14 @@ int create_listeners(struct bind_conf *bc, const struct sockaddr_storage *ss, + */ + void delete_listener(struct listener *listener) + { +- if (listener->state != LI_ASSIGNED) +- return; +- + HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock); +- listener->state = LI_INIT; +- LIST_DEL(&listener->proto_list); +- listener->proto->nb_listeners--; +- HA_ATOMIC_SUB(&jobs, 1); +- HA_ATOMIC_SUB(&listeners, 1); ++ if (listener->state == LI_ASSIGNED) { ++ listener->state = LI_INIT; ++ LIST_DEL(&listener->proto_list); ++ listener->proto->nb_listeners--; ++ HA_ATOMIC_SUB(&jobs, 1); ++ HA_ATOMIC_SUB(&listeners, 1); ++ } + HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock); + } + +@@ -443,6 +441,7 @@ void listener_accept(int fd) + struct listener *l = fdtab[fd].owner; + struct proxy *p; + int max_accept; ++ int next_conn = 0; + int expire; + int cfd; + int ret; +@@ -454,8 +453,6 @@ void listener_accept(int fd) + return; + p = l->bind_conf->frontend; + max_accept = l->maxaccept ? l->maxaccept : 1; +- if (HA_SPIN_TRYLOCK(LISTENER_LOCK, &l->lock)) +- return; + + if (!(l->options & LI_O_UNLIMITED) && global.sps_lim) { + int max = freq_ctr_remain(&global.sess_per_sec, global.sps_lim, 0); +@@ -515,11 +512,29 @@ void listener_accept(int fd) + * worst case. If we fail due to system limits or temporary resource + * shortage, we try again 100ms later in the worst case. + */ +- while (l->nbconn < l->maxconn && max_accept--) { ++ for (; max_accept-- > 0; next_conn = 0) { + struct sockaddr_storage addr; + socklen_t laddr = sizeof(addr); + unsigned int count; + ++ /* pre-increase the number of connections without going too far */ ++ do { ++ count = l->nbconn; ++ if (count >= l->maxconn) { ++ /* the listener was marked full or another ++ * thread is going to do it. ++ */ ++ next_conn = 0; ++ goto end; ++ } ++ next_conn = count + 1; ++ } while (!HA_ATOMIC_CAS(&l->nbconn, &count, next_conn)); ++ ++ if (next_conn == l->maxconn) { ++ /* we filled it, mark it full */ ++ listener_full(l); ++ } ++ + if (unlikely(actconn >= global.maxconn) && !(l->options & LI_O_UNLIMITED)) { + limit_listener(l, &global_listener_queue); + task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */ +@@ -562,6 +577,7 @@ void listener_accept(int fd) + goto transient_error; + case EINTR: + case ECONNABORTED: ++ HA_ATOMIC_SUB(&l->nbconn, 1); + continue; + case ENFILE: + if (p) +@@ -588,26 +604,34 @@ void listener_accept(int fd) + } + } + ++ /* The connection was accepted, it must be counted as such */ ++ if (l->counters) ++ HA_ATOMIC_UPDATE_MAX(&l->counters->conn_max, next_conn); ++ ++ if (!(l->options & LI_O_UNLIMITED)) { ++ count = update_freq_ctr(&global.conn_per_sec, 1); ++ HA_ATOMIC_UPDATE_MAX(&global.cps_max, count); ++ HA_ATOMIC_ADD(&actconn, 1); ++ } ++ + if (unlikely(cfd >= global.maxsock)) { + send_log(p, LOG_EMERG, + "Proxy %s reached the configured maximum connection limit. Please check the global 'maxconn' value.\n", + p->id); ++ if (!(l->options & LI_O_UNLIMITED)) ++ HA_ATOMIC_SUB(&actconn, 1); + close(cfd); + limit_listener(l, &global_listener_queue); + task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */ + goto end; + } + +- /* increase the per-process number of cumulated connections */ +- if (!(l->options & LI_O_UNLIMITED)) { +- count = update_freq_ctr(&global.conn_per_sec, 1); +- HA_ATOMIC_UPDATE_MAX(&global.cps_max, count); +- HA_ATOMIC_ADD(&actconn, 1); +- } +- +- count = HA_ATOMIC_ADD(&l->nbconn, 1); +- if (l->counters) +- HA_ATOMIC_UPDATE_MAX(&l->counters->conn_max, count); ++ /* past this point, l->accept() will automatically decrement ++ * l->nbconn and actconn once done. Setting next_conn=0 allows ++ * the error path not to rollback on nbconn. It's more convenient ++ * than duplicating all exit labels. ++ */ ++ next_conn = 0; + + ret = l->accept(l, cfd, &addr); + if (unlikely(ret <= 0)) { +@@ -622,7 +646,9 @@ void listener_accept(int fd) + goto transient_error; + } + +- /* increase the per-process number of cumulated connections */ ++ /* increase the per-process number of cumulated sessions, this ++ * may only be done once l->accept() has accepted the connection. ++ */ + if (!(l->options & LI_O_UNLIMITED)) { + count = update_freq_ctr(&global.sess_per_sec, 1); + HA_ATOMIC_UPDATE_MAX(&global.sps_max, count); +@@ -634,7 +660,7 @@ void listener_accept(int fd) + } + #endif + +- } /* end of while (max_accept--) */ ++ } /* end of for (max_accept--) */ + + /* we've exhausted max_accept, so there is no need to poll again */ + stop: +@@ -649,10 +675,21 @@ void listener_accept(int fd) + limit_listener(l, &global_listener_queue); + task_schedule(global_listener_queue_task, tick_first(expire, global_listener_queue_task->expire)); + end: +- if (l->nbconn >= l->maxconn) +- listener_full(l); ++ if (next_conn) ++ HA_ATOMIC_SUB(&l->nbconn, 1); + +- HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock); ++ if (l->nbconn < l->maxconn && l->state == LI_FULL) { ++ /* at least one thread has to this when quitting */ ++ resume_listener(l); ++ ++ /* Dequeues all of the listeners waiting for a resource */ ++ if (!LIST_ISEMPTY(&global_listener_queue)) ++ dequeue_all_listeners(&global_listener_queue); ++ ++ if (!LIST_ISEMPTY(&p->listener_queue) && ++ (!p->fe_sps_lim || freq_ctr_remain(&p->fe_sess_per_sec, p->fe_sps_lim, 0) > 0)) ++ dequeue_all_listeners(&p->listener_queue); ++ } + } + + /* Notify the listener that a connection initiated from it was released. This diff --git a/net/haproxy/patches/037-BUG-MEDIUM-listener-use-a-self-locked-list-for-the-dequeue-lists.patch b/net/haproxy/patches/037-BUG-MEDIUM-listener-use-a-self-locked-list-for-the-dequeue-lists.patch new file mode 100644 index 0000000000..f565f92821 --- /dev/null +++ b/net/haproxy/patches/037-BUG-MEDIUM-listener-use-a-self-locked-list-for-the-dequeue-lists.patch @@ -0,0 +1,247 @@ +commit e75d8efec35de2d22d14bf03f4aa9b8490658788 +Author: Willy Tarreau +Date: Thu Feb 28 10:27:18 2019 +0100 + + BUG/MEDIUM: listener: use a self-locked list for the dequeue lists + + There is a very difficult to reproduce race in the listener's accept + code, which is much easier to reproduce once connection limits are + properly enforced. It's an ABBA lock issue : + + - the following functions take l->lock then lq_lock : + disable_listener, pause_listener, listener_full, limit_listener, + do_unbind_listener + + - the following ones take lq_lock then l->lock : + resume_listener, dequeue_all_listener + + This is because __resume_listener() only takes the listener's lock + and expects to be called with lq_lock held. The problem can easily + happen when listener_full() and limit_listener() are called a lot + while in parallel another thread releases sessions for the same + listener using listener_release() which in turn calls resume_listener(). + + This scenario is more prevalent in 2.0-dev since the removal of the + accept lock in listener_accept(). However in 1.9 and before, a different + but extremely unlikely scenario can happen : + + thread1 thread2 + ............................ enter listener_accept() + limit_listener() + ............................ long pause before taking the lock + session_free() + dequeue_all_listeners() + lock(lq_lock) [1] + ............................ try_lock(l->lock) [2] + __resume_listener() + spin_lock(l->lock) =>WAIT[2] + ............................ accept() + l->accept() + nbconn==maxconn => + listener_full() + state==LI_LIMITED => + lock(lq_lock) =>DEADLOCK[1]! + + In practice it is almost impossible to trigger it because it requires + to limit both on the listener's maxconn and the frontend's rate limit, + at the same time, and to release the listener when the connection rate + goes below the limit between poll() returns the FD and the lock is + taken (a few nanoseconds). But maybe with threads competing on the + same core it has more chances to appear. + + This patch removes the lq_lock and replaces it with a lockless queue + for the listener's wait queue (well, technically speaking a self-locked + queue) brought by commit a8434ec14 ("MINOR: lists: Implement locked + variations.") and its few subsequent fixes. This relieves us from the + need of the lq_lock and removes the deadlock. It also gets rid of the + distinction between __resume_listener() and resume_listener() since the + only difference was the lq_lock. All listener removals from the list + are now unconditional to avoid races on the state. It's worth noting + that the list used to never be initialized and that it used to work + only thanks to the state tests, so the initialization has now been + added. + + This patch must carefully be backported to 1.9 and very likely 1.8. + It is mandatory to be careful about replacing all manipulations of + l->wait_queue, global.listener_queue and p->listener_queue. + + (cherry picked from commit 01abd025084b4fe50e84189d1a83499cbf4825ed) + [wt: there are some suspicions that this bug might have been hit in + 1.9 so it's about time to backport this fix. All occurrences of the + elements switched to self-locked list were inspected and none of + them differred from 2.0so that the patch doesn't need to be adjusted] + Signed-off-by: Willy Tarreau + (cherry picked from commit 434c7f8419fc38b6ced1945fa4ce6e84c063e835) + Signed-off-by: Christopher Faulet + +diff --git a/include/common/hathreads.h b/include/common/hathreads.h +index 8a738aaf..8134839a 100644 +--- a/include/common/hathreads.h ++++ b/include/common/hathreads.h +@@ -344,7 +344,6 @@ enum lock_label { + TASK_WQ_LOCK, + POOL_LOCK, + LISTENER_LOCK, +- LISTENER_QUEUE_LOCK, + PROXY_LOCK, + SERVER_LOCK, + UPDATED_SERVERS_LOCK, +@@ -467,7 +466,6 @@ static inline const char *lock_label(enum lock_label label) + case TASK_WQ_LOCK: return "TASK_WQ"; + case POOL_LOCK: return "POOL"; + case LISTENER_LOCK: return "LISTENER"; +- case LISTENER_QUEUE_LOCK: return "LISTENER_QUEUE"; + case PROXY_LOCK: return "PROXY"; + case SERVER_LOCK: return "SERVER"; + case UPDATED_SERVERS_LOCK: return "UPDATED_SERVERS"; +diff --git a/src/listener.c b/src/listener.c +index 9ef3d5e8..dab07a5e 100644 +--- a/src/listener.c ++++ b/src/listener.c +@@ -39,9 +39,6 @@ + #include + #include + +- /* listner_queue lock (same for global and per proxy queues) */ +-__decl_hathreads(static HA_SPINLOCK_T lq_lock); +- + /* List head of all known bind keywords */ + static struct bind_kw_list bind_keywords = { + .list = LIST_HEAD_INIT(bind_keywords.list) +@@ -94,11 +91,7 @@ static void disable_listener(struct listener *listener) + goto end; + if (listener->state == LI_READY) + fd_stop_recv(listener->fd); +- if (listener->state == LI_LIMITED) { +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- LIST_DEL(&listener->wait_queue); +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- } ++ LIST_DEL_LOCKED(&listener->wait_queue); + listener->state = LI_LISTEN; + end: + HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock); +@@ -134,11 +127,7 @@ int pause_listener(struct listener *l) + goto end; + } + +- if (l->state == LI_LIMITED) { +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- LIST_DEL(&l->wait_queue); +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- } ++ LIST_DEL_LOCKED(&l->wait_queue); + + fd_stop_recv(l->fd); + l->state = LI_PAUSED; +@@ -157,7 +146,7 @@ int pause_listener(struct listener *l) + * stopped it. If the resume fails, 0 is returned and an error might be + * displayed. + */ +-static int __resume_listener(struct listener *l) ++int resume_listener(struct listener *l) + { + int ret = 1; + +@@ -199,8 +188,7 @@ static int __resume_listener(struct listener *l) + if (l->state == LI_READY) + goto end; + +- if (l->state == LI_LIMITED) +- LIST_DEL(&l->wait_queue); ++ LIST_DEL_LOCKED(&l->wait_queue); + + if (l->nbconn >= l->maxconn) { + l->state = LI_FULL; +@@ -214,16 +202,6 @@ static int __resume_listener(struct listener *l) + return ret; + } + +-int resume_listener(struct listener *l) +-{ +- int ret; +- +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- ret = __resume_listener(l); +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- return ret; +-} +- + /* Marks a ready listener as full so that the stream code tries to re-enable + * it upon next close() using resume_listener(). + */ +@@ -231,11 +209,7 @@ static void listener_full(struct listener *l) + { + HA_SPIN_LOCK(LISTENER_LOCK, &l->lock); + if (l->state >= LI_READY) { +- if (l->state == LI_LIMITED) { +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- LIST_DEL(&l->wait_queue); +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- } ++ LIST_DEL_LOCKED(&l->wait_queue); + if (l->state != LI_FULL) { + fd_stop_recv(l->fd); + l->state = LI_FULL; +@@ -251,9 +225,7 @@ static void limit_listener(struct listener *l, struct list *list) + { + HA_SPIN_LOCK(LISTENER_LOCK, &l->lock); + if (l->state == LI_READY) { +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- LIST_ADDQ(list, &l->wait_queue); +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); ++ LIST_ADDQ_LOCKED(list, &l->wait_queue); + fd_stop_recv(l->fd); + l->state = LI_LIMITED; + } +@@ -292,17 +264,14 @@ int disable_all_listeners(struct protocol *proto) + /* Dequeues all of the listeners waiting for a resource in wait queue . */ + void dequeue_all_listeners(struct list *list) + { +- struct listener *listener, *l_back; ++ struct listener *listener; + +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- list_for_each_entry_safe(listener, l_back, list, wait_queue) { ++ while ((listener = LIST_POP_LOCKED(list, struct listener *, wait_queue))) { + /* This cannot fail because the listeners are by definition in +- * the LI_LIMITED state. The function also removes the entry +- * from the queue. ++ * the LI_LIMITED state. + */ +- __resume_listener(listener); ++ resume_listener(listener); + } +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); + } + + /* Must be called with the lock held. Depending on value, it does +@@ -313,11 +282,7 @@ void do_unbind_listener(struct listener *listener, int do_close) + if (listener->state == LI_READY) + fd_stop_recv(listener->fd); + +- if (listener->state == LI_LIMITED) { +- HA_SPIN_LOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- LIST_DEL(&listener->wait_queue); +- HA_SPIN_UNLOCK(LISTENER_QUEUE_LOCK, &lq_lock); +- } ++ LIST_DEL_LOCKED(&listener->wait_queue); + + if (listener->state >= LI_PAUSED) { + if (do_close) { +@@ -399,6 +364,7 @@ int create_listeners(struct bind_conf *bc, const struct sockaddr_storage *ss, + + l->fd = fd; + memcpy(&l->addr, ss, sizeof(*ss)); ++ LIST_INIT(&l->wait_queue); + l->state = LI_INIT; + + proto->add(l, port); +@@ -1039,7 +1005,6 @@ static void __listener_init(void) + sample_register_fetches(&smp_kws); + acl_register_keywords(&acl_kws); + bind_register_keywords(&bind_kws); +- HA_SPIN_INIT(&lq_lock); + } + + /* diff --git a/net/haproxy/patches/038-BUG-MEDIUM-listener-make-sure-the-listener-never-accepts-too-many-conns.patch b/net/haproxy/patches/038-BUG-MEDIUM-listener-make-sure-the-listener-never-accepts-too-many-conns.patch new file mode 100644 index 0000000000..a1ca73b2e4 --- /dev/null +++ b/net/haproxy/patches/038-BUG-MEDIUM-listener-make-sure-the-listener-never-accepts-too-many-conns.patch @@ -0,0 +1,217 @@ +commit c98cdf7cc755c579a8b9cceee4809089267615ce +Author: Willy Tarreau +Date: Wed Feb 27 19:32:32 2019 +0100 + + BUG/MEDIUM: listener: make sure the listener never accepts too many conns + + We were not checking p->feconn nor the global actconn soon enough. In + older versions this could result in a frontend accepting more connections + than allowed by its maxconn or the global maxconn, exactly N-1 extra + connections where N is the number of threads, provided each of these + threads were running a different listener. But with the lock removal, + it became worse, the excess could be the listener's maxconn multiplied + by the number of threads. Among the nasty side effect was that LI_FULL + could be removed while the limit was still over and in some cases the + polling on the socket was no re-enabled. + + This commit takes care of updating and checking p->feconn and the global + actconn *before* processing the connection, so that the listener can be + turned off before accepting the socket if needed. This requires to move + some of the bookkeeping operations form session to listen, which totally + makes sense in this context. + + Now the limits are properly respected, even if a listener's maxconn is + over a frontend's. This only applies on top of the listener lock removal + series and doesn't have to be backported. + + (cherry picked from commit 82c9789ac4f0cba9a74d16c1b730fc64e1f95a6e) + [wt: backported since it fixes the previous patch set] + Signed-off-by: Willy Tarreau + (cherry picked from commit f54a86a229e1ee4b256d5614c0a65924f447df09) + Signed-off-by: Christopher Faulet + +diff --git a/src/listener.c b/src/listener.c +index dab07a5e..68c84fbe 100644 +--- a/src/listener.c ++++ b/src/listener.c +@@ -408,6 +408,8 @@ void listener_accept(int fd) + struct proxy *p; + int max_accept; + int next_conn = 0; ++ int next_feconn = 0; ++ int next_actconn = 0; + int expire; + int cfd; + int ret; +@@ -478,12 +480,15 @@ void listener_accept(int fd) + * worst case. If we fail due to system limits or temporary resource + * shortage, we try again 100ms later in the worst case. + */ +- for (; max_accept-- > 0; next_conn = 0) { ++ for (; max_accept-- > 0; next_conn = next_feconn = next_actconn = 0) { + struct sockaddr_storage addr; + socklen_t laddr = sizeof(addr); + unsigned int count; + +- /* pre-increase the number of connections without going too far */ ++ /* pre-increase the number of connections without going too far. ++ * We process the listener, then the proxy, then the process. ++ * We know which ones to unroll based on the next_xxx value. ++ */ + do { + count = l->nbconn; + if (count >= l->maxconn) { +@@ -501,15 +506,42 @@ void listener_accept(int fd) + listener_full(l); + } + +- if (unlikely(actconn >= global.maxconn) && !(l->options & LI_O_UNLIMITED)) { +- limit_listener(l, &global_listener_queue); +- task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */ +- goto end; ++ if (p) { ++ do { ++ count = p->feconn; ++ if (count >= p->maxconn) { ++ /* the frontend was marked full or another ++ * thread is going to do it. ++ */ ++ next_feconn = 0; ++ goto end; ++ } ++ next_feconn = count + 1; ++ } while (!HA_ATOMIC_CAS(&p->feconn, &count, next_feconn)); ++ ++ if (unlikely(next_feconn == p->maxconn)) { ++ /* we just filled it */ ++ limit_listener(l, &p->listener_queue); ++ } + } + +- if (unlikely(p && p->feconn >= p->maxconn)) { +- limit_listener(l, &p->listener_queue); +- goto end; ++ if (!(l->options & LI_O_UNLIMITED)) { ++ do { ++ count = actconn; ++ if (count >= global.maxconn) { ++ /* the process was marked full or another ++ * thread is going to do it. ++ */ ++ next_actconn = 0; ++ goto end; ++ } ++ next_actconn = count + 1; ++ } while (!HA_ATOMIC_CAS(&actconn, &count, next_actconn)); ++ ++ if (unlikely(next_actconn == global.maxconn)) { ++ limit_listener(l, &global_listener_queue); ++ task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */ ++ } + } + + #ifdef USE_ACCEPT4 +@@ -544,6 +576,10 @@ void listener_accept(int fd) + case EINTR: + case ECONNABORTED: + HA_ATOMIC_SUB(&l->nbconn, 1); ++ if (p) ++ HA_ATOMIC_SUB(&p->feconn, 1); ++ if (!(l->options & LI_O_UNLIMITED)) ++ HA_ATOMIC_SUB(&actconn, 1); + continue; + case ENFILE: + if (p) +@@ -574,18 +610,20 @@ void listener_accept(int fd) + if (l->counters) + HA_ATOMIC_UPDATE_MAX(&l->counters->conn_max, next_conn); + ++ if (p) ++ HA_ATOMIC_UPDATE_MAX(&p->fe_counters.conn_max, next_feconn); ++ ++ proxy_inc_fe_conn_ctr(l, p); ++ + if (!(l->options & LI_O_UNLIMITED)) { + count = update_freq_ctr(&global.conn_per_sec, 1); + HA_ATOMIC_UPDATE_MAX(&global.cps_max, count); +- HA_ATOMIC_ADD(&actconn, 1); + } + + if (unlikely(cfd >= global.maxsock)) { + send_log(p, LOG_EMERG, + "Proxy %s reached the configured maximum connection limit. Please check the global 'maxconn' value.\n", + p->id); +- if (!(l->options & LI_O_UNLIMITED)) +- HA_ATOMIC_SUB(&actconn, 1); + close(cfd); + limit_listener(l, &global_listener_queue); + task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */ +@@ -593,11 +631,13 @@ void listener_accept(int fd) + } + + /* past this point, l->accept() will automatically decrement +- * l->nbconn and actconn once done. Setting next_conn=0 allows +- * the error path not to rollback on nbconn. It's more convenient +- * than duplicating all exit labels. ++ * l->nbconn, feconn and actconn once done. Setting next_*conn=0 ++ * allows the error path not to rollback on nbconn. It's more ++ * convenient than duplicating all exit labels. + */ + next_conn = 0; ++ next_feconn = 0; ++ next_actconn = 0; + + ret = l->accept(l, cfd, &addr); + if (unlikely(ret <= 0)) { +@@ -644,7 +684,14 @@ void listener_accept(int fd) + if (next_conn) + HA_ATOMIC_SUB(&l->nbconn, 1); + +- if (l->nbconn < l->maxconn && l->state == LI_FULL) { ++ if (p && next_feconn) ++ HA_ATOMIC_SUB(&p->feconn, 1); ++ ++ if (next_actconn) ++ HA_ATOMIC_SUB(&actconn, 1); ++ ++ if ((l->state == LI_FULL && l->nbconn < l->maxconn) || ++ (l->state == LI_LIMITED && ((!p || p->feconn < p->maxconn) && (actconn < global.maxconn)))) { + /* at least one thread has to this when quitting */ + resume_listener(l); + +@@ -668,8 +715,11 @@ void listener_release(struct listener *l) + + if (!(l->options & LI_O_UNLIMITED)) + HA_ATOMIC_SUB(&actconn, 1); ++ if (fe) ++ HA_ATOMIC_SUB(&fe->feconn, 1); + HA_ATOMIC_SUB(&l->nbconn, 1); +- if (l->state == LI_FULL) ++ ++ if (l->state == LI_FULL || l->state == LI_LIMITED) + resume_listener(l); + + /* Dequeues all of the listeners waiting for a resource */ +diff --git a/src/session.c b/src/session.c +index b91d67ee..c1515261 100644 +--- a/src/session.c ++++ b/src/session.c +@@ -53,10 +53,6 @@ struct session *session_new(struct proxy *fe, struct listener *li, enum obj_type + vars_init(&sess->vars, SCOPE_SESS); + sess->task = NULL; + sess->t_handshake = -1; /* handshake not done yet */ +- HA_ATOMIC_UPDATE_MAX(&fe->fe_counters.conn_max, +- HA_ATOMIC_ADD(&fe->feconn, 1)); +- if (li) +- proxy_inc_fe_conn_ctr(li, fe); + HA_ATOMIC_ADD(&totalconn, 1); + HA_ATOMIC_ADD(&jobs, 1); + } +@@ -65,7 +61,6 @@ struct session *session_new(struct proxy *fe, struct listener *li, enum obj_type + + void session_free(struct session *sess) + { +- HA_ATOMIC_SUB(&sess->fe->feconn, 1); + if (sess->listener) + listener_release(sess->listener); + session_store_counters(sess); diff --git a/net/haproxy/patches/039-BUILD-MINOR-listener-Silent-a-few-signedness-warnings.patch b/net/haproxy/patches/039-BUILD-MINOR-listener-Silent-a-few-signedness-warnings.patch new file mode 100644 index 0000000000..02a851ea78 --- /dev/null +++ b/net/haproxy/patches/039-BUILD-MINOR-listener-Silent-a-few-signedness-warnings.patch @@ -0,0 +1,37 @@ +commit 06ffb1175eb87684eaccb4fd7ac9d4936ca9b8f7 +Author: David Carlier +Date: Wed Mar 27 16:08:42 2019 +0000 + + BUILD/MINOR: listener: Silent a few signedness warnings. + + Silenting couple of warnings related to signedness, due to a mismatch of + signed and unsigned ints with l->nbconn, actconn and p->feconn. + + (cherry picked from commit 5671662f08536c979ff91bc46f94d086bf286540) + [wt: s/_HA/HA/] + Signed-off-by: Willy Tarreau + (cherry picked from commit f5e9bf696b0b46c140d742cee23d0d54df66bb2f) + Signed-off-by: Christopher Faulet + +diff --git a/src/listener.c b/src/listener.c +index 68c84fbe..821c931a 100644 +--- a/src/listener.c ++++ b/src/listener.c +@@ -499,7 +499,7 @@ void listener_accept(int fd) + goto end; + } + next_conn = count + 1; +- } while (!HA_ATOMIC_CAS(&l->nbconn, &count, next_conn)); ++ } while (!HA_ATOMIC_CAS(&l->nbconn, (int *)&count, next_conn)); + + if (next_conn == l->maxconn) { + /* we filled it, mark it full */ +@@ -536,7 +536,7 @@ void listener_accept(int fd) + goto end; + } + next_actconn = count + 1; +- } while (!HA_ATOMIC_CAS(&actconn, &count, next_actconn)); ++ } while (!HA_ATOMIC_CAS(&actconn, (int *)&count, next_actconn)); + + if (unlikely(next_actconn == global.maxconn)) { + limit_listener(l, &global_listener_queue); diff --git a/net/haproxy/patches/040-MINOR-skip-get_gmtime-where-tm-is-unused.patch b/net/haproxy/patches/040-MINOR-skip-get_gmtime-where-tm-is-unused.patch new file mode 100644 index 0000000000..fab1cb3012 --- /dev/null +++ b/net/haproxy/patches/040-MINOR-skip-get_gmtime-where-tm-is-unused.patch @@ -0,0 +1,28 @@ +commit 5d9e3238ae9474051b2020a04af2cbb11b613f98 +Author: Robin H. Johnson +Date: Wed Apr 10 21:08:15 2019 +0000 + + MINOR: skip get_gmtime where tm is unused + + For LOG_FMT_TS (%Ts), the tm variable is not used, so save some cycles + on the call to get_gmtime. + + Backport: 1.9 1.8 + Signed-off-by: Robin H. Johnson + (cherry picked from commit 543d4507ca4ddd9ece5eb4e869b20ee1d2afedac) + Signed-off-by: Willy Tarreau + (cherry picked from commit 0ab133673a28fb91679f2b8471ed13ce265aa8a6) + Signed-off-by: Christopher Faulet + +diff --git a/src/log.c b/src/log.c +index 9c112255..313fa55d 100644 +--- a/src/log.c ++++ b/src/log.c +@@ -1651,7 +1651,6 @@ int build_logline(struct stream *s, char *dst, size_t maxsize, struct list *list + break; + + case LOG_FMT_TS: // %Ts +- get_gmtime(s->logs.accept_date.tv_sec, &tm); + if (tmp->options & LOG_OPT_HEXA) { + iret = snprintf(tmplog, dst + maxsize - tmplog, "%04X", (unsigned int)s->logs.accept_date.tv_sec); + if (iret < 0 || iret > dst + maxsize - tmplog) diff --git a/net/haproxy/patches/041-BUG-MAJOR-http_fetch-Get-the-channel-depending-on-the-keyword-used.patch b/net/haproxy/patches/041-BUG-MAJOR-http_fetch-Get-the-channel-depending-on-the-keyword-used.patch new file mode 100644 index 0000000000..3f8fb1c32e --- /dev/null +++ b/net/haproxy/patches/041-BUG-MAJOR-http_fetch-Get-the-channel-depending-on-the-keyword-used.patch @@ -0,0 +1,976 @@ +commit b05ee4aa74a95be49c78198ca601000b47c93da2 +Author: Christopher Faulet +Date: Wed Apr 17 12:02:59 2019 +0200 + + BUG/MAJOR: http_fetch: Get the channel depending on the keyword used + + All HTTP samples are buggy because the channel tested in the prefetch functions + (HTX and legacy HTTP) is chosen depending on the sample direction and not the + keyword really used. It means the request channel is used if the sample is + called during the request analysis and the response channel is used if it is + called during the response analysis, regardless the sample really called. For + instance, if you use the sample "req.ver" in an http-response rule, the response + channel will be prefeched because it is called during the response analysis, + while the request channel should have been used instead. So some assumptions on + the validity of the sample may be made on the wrong channel. It is the first + bug. + + Then the same error is done in some samples themselves. So fetches are performed + on the wrong channel. For instance, the header extraction (req.fhdr, res.fhdr, + req.hdr, res.hdr...). If the sample "req.hdr" is used in an http-response rule, + then the matching is done on the response headers and not the request ones. It + is the second bug. + + Finally, the last one but not the least, in some samples, the right channel is + used. But because the prefetch was done on the wrong one, this channel may be in + a undefined state. For instance, using the sample "req.ver" in an http-response + rule leads to a matching on a posibility released buffer. + + To fix all these bugs, the right channel is now chosen in sample fetches, before + the prefetch. If the same function is used to fetch requests and responses + elements, then the keyword is used to choose the right one. This channel is then + used by the functions smp_prefetch_htx() and smp_prefetch_http(). Of course, it + is also used by the samples themselves to extract information. + + This patch must be backported to all supported versions. For version 1.8 and + priors, it must be totally refactored. First because there is no HTX into these + versions. Then the buffers API has changed in HAProxy 1.9. The files + http_fetch.{ch} doesn't exist on old versions. + + (cherry picked from commit 89dc49935997856dd4f864b654d3601107ec1967) + Signed-off-by: Christopher Faulet + (cherry picked from commit a89ca0b50b6b30ab75fbb7193a1132c0f89520fc) + [cf: Changes made in src/proto_http.c because src/http_fetch.c doesn't exist] + Signed-off-by: Christopher Faulet + +diff --git a/include/proto/proto_http.h b/include/proto/proto_http.h +index 50efff4c..a2b365da 100644 +--- a/include/proto/proto_http.h ++++ b/include/proto/proto_http.h +@@ -132,7 +132,7 @@ struct action_kw *action_http_res_custom(const char *kw); + int val_hdr(struct arg *arg, char **err_msg); + + int smp_prefetch_http(struct proxy *px, struct stream *s, unsigned int opt, +- const struct arg *args, struct sample *smp, int req_vol); ++ const struct channel *chn, struct sample *smp, int req_vol); + + enum act_return http_action_req_capture_by_id(struct act_rule *rule, struct proxy *px, + struct session *sess, struct stream *s, int flags); +@@ -144,11 +144,11 @@ int parse_qvalue(const char *qvalue, const char **end); + /* Note: these functions *do* modify the sample. Even in case of success, at + * least the type and uint value are modified. + */ +-#define CHECK_HTTP_MESSAGE_FIRST() \ +- do { int r = smp_prefetch_http(smp->px, smp->strm, smp->opt, args, smp, 1); if (r <= 0) return r; } while (0) ++#define CHECK_HTTP_MESSAGE_FIRST(chn) \ ++ do { int r = smp_prefetch_http(smp->px, smp->strm, smp->opt, (chn), smp, 1); if (r <= 0) return r; } while (0) + +-#define CHECK_HTTP_MESSAGE_FIRST_PERM() \ +- do { int r = smp_prefetch_http(smp->px, smp->strm, smp->opt, args, smp, 0); if (r <= 0) return r; } while (0) ++#define CHECK_HTTP_MESSAGE_FIRST_PERM(chn) \ ++ do { int r = smp_prefetch_http(smp->px, smp->strm, smp->opt, (chn), smp, 0); if (r <= 0) return r; } while (0) + + static inline void http_req_keywords_register(struct action_kw_list *kw_list) + { +diff --git a/src/hlua.c b/src/hlua.c +index 93cb86d2..d40c012a 100644 +--- a/src/hlua.c ++++ b/src/hlua.c +@@ -6458,7 +6458,7 @@ static int hlua_applet_http_init(struct appctx *ctx, struct proxy *px, struct st + const char *error; + + /* Wait for a full HTTP request. */ +- if (!smp_prefetch_http(px, strm, 0, NULL, &smp, 0)) { ++ if (!smp_prefetch_http(px, strm, 0, req, &smp, 0)) { + if (smp.flags & SMP_F_MAY_CHANGE) + return -1; + return 0; +diff --git a/src/proto_http.c b/src/proto_http.c +index 8b087c5b..9beaa137 100644 +--- a/src/proto_http.c ++++ b/src/proto_http.c +@@ -9447,6 +9447,8 @@ struct redirect_rule *http_parse_redirect_rule(const char *file, int linenum, st + /************************************************************************/ + /* The code below is dedicated to ACL parsing and matching */ + /************************************************************************/ ++#define SMP_REQ_CHN(smp) (smp->strm ? &smp->strm->req : NULL) ++#define SMP_RES_CHN(smp) (smp->strm ? &smp->strm->res : NULL) + + + /* This function ensures that the prerequisites for an L7 fetch are ready, +@@ -9463,7 +9465,7 @@ struct redirect_rule *http_parse_redirect_rule(const char *file, int linenum, st + * 1 if an HTTP message is ready + */ + int smp_prefetch_http(struct proxy *px, struct stream *s, unsigned int opt, +- const struct arg *args, struct sample *smp, int req_vol) ++ const struct channel *chn, struct sample *smp, int req_vol) + { + struct http_txn *txn; + struct http_msg *msg; +@@ -9472,7 +9474,7 @@ int smp_prefetch_http(struct proxy *px, struct stream *s, unsigned int opt, + * initialization (eg: tcp-request connection), so this function is the + * one responsible for guarding against this case for all HTTP users. + */ +- if (!s) ++ if (!s || !chn) + return 0; + + if (!s->txn) { +@@ -9481,78 +9483,78 @@ int smp_prefetch_http(struct proxy *px, struct stream *s, unsigned int opt, + http_init_txn(s); + } + txn = s->txn; +- msg = &txn->req; + +- /* Check for a dependency on a request */ + smp->data.type = SMP_T_BOOL; + +- if ((opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) { +- /* If the buffer does not leave enough free space at the end, +- * we must first realign it. +- */ +- if (s->req.buf->p > s->req.buf->data && +- s->req.buf->i + s->req.buf->p > s->req.buf->data + s->req.buf->size - global.tune.maxrewrite) +- buffer_slow_realign(s->req.buf); +- +- if (unlikely(txn->req.msg_state < HTTP_MSG_BODY)) { +- if (msg->msg_state == HTTP_MSG_ERROR) +- return 0; ++ if (chn->flags & CF_ISRESP) { ++ /* Check for a dependency on a response */ ++ if (txn->rsp.msg_state < HTTP_MSG_BODY) { ++ smp->flags |= SMP_F_MAY_CHANGE; ++ return 0; ++ } ++ goto end; ++ } + +- /* Try to decode HTTP request */ +- if (likely(msg->next < s->req.buf->i)) +- http_msg_analyzer(msg, &txn->hdr_idx); ++ /* Check for a dependency on a request */ ++ msg = &txn->req; + +- /* Still no valid request ? */ +- if (unlikely(msg->msg_state < HTTP_MSG_BODY)) { +- if ((msg->msg_state == HTTP_MSG_ERROR) || +- buffer_full(s->req.buf, global.tune.maxrewrite)) { +- return 0; +- } +- /* wait for final state */ +- smp->flags |= SMP_F_MAY_CHANGE; +- return 0; +- } ++ if (req_vol && (smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_RES) { ++ return 0; /* data might have moved and indexes changed */ ++ } + +- /* OK we just got a valid HTTP request. We have some minor +- * preparation to perform so that further checks can rely +- * on HTTP tests. +- */ ++ /* If the buffer does not leave enough free space at the end, we must ++ * first realign it. ++ */ ++ if (chn->buf->p > chn->buf->data && ++ chn->buf->i + chn->buf->p > chn->buf->data + chn->buf->size - global.tune.maxrewrite) ++ buffer_slow_realign(chn->buf); + +- /* If the request was parsed but was too large, we must absolutely +- * return an error so that it is not processed. At the moment this +- * cannot happen, but if the parsers are to change in the future, +- * we want this check to be maintained. +- */ +- if (unlikely(s->req.buf->i + s->req.buf->p > +- s->req.buf->data + s->req.buf->size - global.tune.maxrewrite)) { +- msg->err_state = msg->msg_state; +- msg->msg_state = HTTP_MSG_ERROR; +- smp->data.u.sint = 1; +- return 1; +- } ++ if (unlikely(msg->msg_state < HTTP_MSG_BODY)) { ++ if (msg->msg_state == HTTP_MSG_ERROR) ++ return 0; + +- txn->meth = find_http_meth(msg->chn->buf->p, msg->sl.rq.m_l); +- if (txn->meth == HTTP_METH_GET || txn->meth == HTTP_METH_HEAD) +- s->flags |= SF_REDIRECTABLE; ++ /* Try to decode HTTP request */ ++ if (likely(msg->next < chn->buf->i)) ++ http_msg_analyzer(msg, &txn->hdr_idx); + +- if (unlikely(msg->sl.rq.v_l == 0) && !http_upgrade_v09_to_v10(txn)) ++ /* Still no valid request ? */ ++ if (unlikely(msg->msg_state < HTTP_MSG_BODY)) { ++ if ((msg->msg_state == HTTP_MSG_ERROR) || ++ buffer_full(chn->buf, global.tune.maxrewrite)) { + return 0; ++ } ++ /* wait for final state */ ++ smp->flags |= SMP_F_MAY_CHANGE; ++ return 0; + } + +- if (req_vol && txn->rsp.msg_state != HTTP_MSG_RPBEFORE) { +- return 0; /* data might have moved and indexes changed */ ++ /* OK we just got a valid HTTP message. We have some minor ++ * preparation to perform so that further checks can rely ++ * on HTTP tests. ++ */ ++ ++ /* If the message was parsed but was too large, we must absolutely ++ * return an error so that it is not processed. At the moment this ++ * cannot happen, but if the parsers are to change in the future, ++ * we want this check to be maintained. ++ */ ++ if (unlikely(chn->buf->i + chn->buf->p > ++ chn->buf->data + chn->buf->size - global.tune.maxrewrite)) { ++ msg->err_state = msg->msg_state; ++ msg->msg_state = HTTP_MSG_ERROR; ++ smp->data.u.sint = 1; ++ return 1; + } + +- /* otherwise everything's ready for the request */ +- } +- else { +- /* Check for a dependency on a response */ +- if (txn->rsp.msg_state < HTTP_MSG_BODY) { +- smp->flags |= SMP_F_MAY_CHANGE; ++ txn->meth = find_http_meth(chn->buf->p, msg->sl.rq.m_l); ++ if (txn->meth == HTTP_METH_GET || txn->meth == HTTP_METH_HEAD) ++ s->flags |= SF_REDIRECTABLE; ++ ++ if (unlikely(msg->sl.rq.v_l == 0) && !http_upgrade_v09_to_v10(txn)) + return 0; +- } + } + ++ end: + /* everything's OK */ + smp->data.u.sint = 1; + return 1; +@@ -9592,19 +9594,21 @@ static int pat_parse_meth(const char *text, struct pattern *pattern, int mflags, + static int + smp_fetch_meth(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + int meth; + struct http_txn *txn; + +- CHECK_HTTP_MESSAGE_FIRST_PERM(); ++ CHECK_HTTP_MESSAGE_FIRST_PERM(chn); + + txn = smp->strm->txn; + meth = txn->meth; + smp->data.type = SMP_T_METH; + smp->data.u.meth.meth = meth; + if (meth == HTTP_METH_OTHER) { +- if (txn->rsp.msg_state != HTTP_MSG_RPBEFORE) ++ if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_RES) { + /* ensure the indexes are not affected */ + return 0; ++ } + smp->flags |= SMP_F_CONST; + smp->data.u.meth.str.len = txn->req.sl.rq.m_l; + smp->data.u.meth.str.str = txn->req.chn->buf->p; +@@ -9646,15 +9650,16 @@ static struct pattern *pat_match_meth(struct sample *smp, struct pattern_expr *e + static int + smp_fetch_rqver(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + char *ptr; + int len; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + len = txn->req.sl.rq.v_l; +- ptr = txn->req.chn->buf->p + txn->req.sl.rq.v; ++ ptr = chn->buf->p + txn->req.sl.rq.v; + + while ((len-- > 0) && (*ptr++ != '/')); + if (len <= 0) +@@ -9671,18 +9676,17 @@ smp_fetch_rqver(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_stver(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_RES_CHN(smp); + struct http_txn *txn; + char *ptr; + int len; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; +- if (txn->rsp.msg_state < HTTP_MSG_BODY) +- return 0; + + len = txn->rsp.sl.st.v_l; +- ptr = txn->rsp.chn->buf->p; ++ ptr = chn->buf->p; + + while ((len-- > 0) && (*ptr++ != '/')); + if (len <= 0) +@@ -9700,18 +9704,19 @@ smp_fetch_stver(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_stcode(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_RES_CHN(smp); + struct http_txn *txn; + char *ptr; + int len; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + if (txn->rsp.msg_state < HTTP_MSG_BODY) + return 0; + + len = txn->rsp.sl.st.c_l; +- ptr = txn->rsp.chn->buf->p + txn->rsp.sl.st.c; ++ ptr = chn->buf->p + txn->rsp.sl.st.c; + + smp->data.type = SMP_T_SINT; + smp->data.u.sint = __strl2ui(ptr, len); +@@ -9746,20 +9751,21 @@ smp_fetch_uniqueid(const struct arg *args, struct sample *smp, const char *kw, v + static int + smp_fetch_hdrs(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_msg *msg; + struct hdr_idx *idx; + struct http_txn *txn; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + idx = &txn->hdr_idx; + msg = &txn->req; + + smp->data.type = SMP_T_STR; +- smp->data.u.str.str = msg->chn->buf->p + hdr_idx_first_pos(idx); ++ smp->data.u.str.str = chn->buf->p + hdr_idx_first_pos(idx); + smp->data.u.str.len = msg->eoh - hdr_idx_first_pos(idx) + 1 + +- (msg->chn->buf->p[msg->eoh] == '\r'); ++ (chn->buf->p[msg->eoh] == '\r'); + + return 1; + } +@@ -9780,7 +9786,7 @@ smp_fetch_hdrs(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_hdrs_bin(const struct arg *args, struct sample *smp, const char *kw, void *private) + { +- struct http_msg *msg; ++ struct channel *chn = SMP_REQ_CHN(smp); + struct chunk *temp; + struct hdr_idx *idx; + const char *cur_ptr, *cur_next, *p; +@@ -9793,7 +9799,7 @@ smp_fetch_hdrs_bin(const struct arg *args, struct sample *smp, const char *kw, v + char *buf; + char *end; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + temp = get_trash_chunk(); + buf = temp->str; +@@ -9801,11 +9807,10 @@ smp_fetch_hdrs_bin(const struct arg *args, struct sample *smp, const char *kw, v + + txn = smp->strm->txn; + idx = &txn->hdr_idx; +- msg = &txn->req; + + /* Build array of headers. */ + old_idx = 0; +- cur_next = msg->chn->buf->p + hdr_idx_first_pos(idx); ++ cur_next = chn->buf->p + hdr_idx_first_pos(idx); + while (1) { + cur_idx = idx->v[old_idx].next; + if (!cur_idx) +@@ -9880,25 +9885,23 @@ smp_fetch_hdrs_bin(const struct arg *args, struct sample *smp, const char *kw, v + static int + smp_fetch_body(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_msg *msg; + unsigned long len; + unsigned long block1; + char *body; + struct chunk *temp; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + +- if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) +- msg = &smp->strm->txn->req; +- else +- msg = &smp->strm->txn->rsp; ++ msg = &smp->strm->txn->req; + + len = http_body_bytes(msg); +- body = b_ptr(msg->chn->buf, -http_data_rewind(msg)); ++ body = b_ptr(chn->buf, -http_data_rewind(msg)); + + block1 = len; +- if (block1 > msg->chn->buf->data + msg->chn->buf->size - body) +- block1 = msg->chn->buf->data + msg->chn->buf->size - body; ++ if (block1 > chn->buf->data + chn->buf->size - body) ++ block1 = chn->buf->data + chn->buf->size - body; + + if (block1 == len) { + /* buffer is not wrapped (or empty) */ +@@ -9911,7 +9914,7 @@ smp_fetch_body(const struct arg *args, struct sample *smp, const char *kw, void + /* buffer is wrapped, we need to defragment it */ + temp = get_trash_chunk(); + memcpy(temp->str, body, block1); +- memcpy(temp->str + block1, msg->chn->buf->data, len - block1); ++ memcpy(temp->str + block1, chn->buf->data, len - block1); + smp->data.type = SMP_T_BIN; + smp->data.u.str.str = temp->str; + smp->data.u.str.len = len; +@@ -9927,15 +9930,12 @@ smp_fetch_body(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_body_len(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_msg *msg; + +- CHECK_HTTP_MESSAGE_FIRST(); +- +- if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) +- msg = &smp->strm->txn->req; +- else +- msg = &smp->strm->txn->rsp; ++ CHECK_HTTP_MESSAGE_FIRST(chn); + ++ msg = &smp->strm->txn->req; + smp->data.type = SMP_T_SINT; + smp->data.u.sint = http_body_bytes(msg); + +@@ -9951,15 +9951,12 @@ smp_fetch_body_len(const struct arg *args, struct sample *smp, const char *kw, v + static int + smp_fetch_body_size(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_msg *msg; + +- CHECK_HTTP_MESSAGE_FIRST(); +- +- if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) +- msg = &smp->strm->txn->req; +- else +- msg = &smp->strm->txn->rsp; ++ CHECK_HTTP_MESSAGE_FIRST(chn); + ++ msg = &smp->strm->txn->req; + smp->data.type = SMP_T_SINT; + smp->data.u.sint = msg->body_len; + +@@ -9972,14 +9969,15 @@ smp_fetch_body_size(const struct arg *args, struct sample *smp, const char *kw, + static int + smp_fetch_url(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + smp->data.type = SMP_T_STR; + smp->data.u.str.len = txn->req.sl.rq.u_l; +- smp->data.u.str.str = txn->req.chn->buf->p + txn->req.sl.rq.u; ++ smp->data.u.str.str = chn->buf->p + txn->req.sl.rq.u; + smp->flags = SMP_F_VOL_1ST | SMP_F_CONST; + return 1; + } +@@ -9987,13 +9985,14 @@ smp_fetch_url(const struct arg *args, struct sample *smp, const char *kw, void * + static int + smp_fetch_url_ip(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + struct sockaddr_storage addr; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; +- url2sa(txn->req.chn->buf->p + txn->req.sl.rq.u, txn->req.sl.rq.u_l, &addr, NULL); ++ url2sa(chn->buf->p + txn->req.sl.rq.u, txn->req.sl.rq.u_l, &addr, NULL); + if (((struct sockaddr_in *)&addr)->sin_family != AF_INET) + return 0; + +@@ -10006,13 +10005,14 @@ smp_fetch_url_ip(const struct arg *args, struct sample *smp, const char *kw, voi + static int + smp_fetch_url_port(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + struct sockaddr_storage addr; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; +- url2sa(txn->req.chn->buf->p + txn->req.sl.rq.u, txn->req.sl.rq.u_l, &addr, NULL); ++ url2sa(chn->buf->p + txn->req.sl.rq.u, txn->req.sl.rq.u_l, &addr, NULL); + if (((struct sockaddr_in *)&addr)->sin_family != AF_INET) + return 0; + +@@ -10032,6 +10032,8 @@ smp_fetch_url_port(const struct arg *args, struct sample *smp, const char *kw, v + static int + smp_fetch_fhdr(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ /* possible keywords: req.fhdr, res.fhdr */ ++ struct channel *chn = ((kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx *ctx = smp->ctx.a[0]; + const struct http_msg *msg; +@@ -10056,10 +10058,9 @@ smp_fetch_fhdr(const struct arg *args, struct sample *smp, const char *kw, void + occ = args[1].data.sint; + } + +- CHECK_HTTP_MESSAGE_FIRST(); +- ++ CHECK_HTTP_MESSAGE_FIRST(chn); + idx = &smp->strm->txn->hdr_idx; +- msg = ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) ? &smp->strm->txn->req : &smp->strm->txn->rsp; ++ msg = (!(chn->flags & CF_ISRESP) ? &smp->strm->txn->req : &smp->strm->txn->rsp); + + if (ctx && !(smp->flags & SMP_F_NOT_LAST)) + /* search for header from the beginning */ +@@ -10089,9 +10090,10 @@ smp_fetch_fhdr(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_fhdr_cnt(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ /* possible keywords: req.fhdr_cnt, res.fhdr_cnt */ ++ struct channel *chn = ((kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx ctx; +- const struct http_msg *msg; + int cnt; + const char *name = NULL; + int len = 0; +@@ -10101,14 +10103,12 @@ smp_fetch_fhdr_cnt(const struct arg *args, struct sample *smp, const char *kw, v + len = args->data.str.len; + } + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + idx = &smp->strm->txn->hdr_idx; +- msg = ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) ? &smp->strm->txn->req : &smp->strm->txn->rsp; +- + ctx.idx = 0; + cnt = 0; +- while (http_find_full_header2(name, len, msg->chn->buf->p, idx, &ctx)) ++ while (http_find_full_header2(name, len, chn->buf->p, idx, &ctx)) + cnt++; + + smp->data.type = SMP_T_SINT; +@@ -10120,24 +10120,24 @@ smp_fetch_fhdr_cnt(const struct arg *args, struct sample *smp, const char *kw, v + static int + smp_fetch_hdr_names(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ /* possible keywords: req.hdr_names, res.hdr_names */ ++ struct channel *chn = ((kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx ctx; +- const struct http_msg *msg; + struct chunk *temp; + char del = ','; + + if (args && args->type == ARGT_STR) + del = *args[0].data.str.str; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + idx = &smp->strm->txn->hdr_idx; +- msg = ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) ? &smp->strm->txn->req : &smp->strm->txn->rsp; + + temp = get_trash_chunk(); + + ctx.idx = 0; +- while (http_find_next_header(msg->chn->buf->p, idx, &ctx)) { ++ while (http_find_next_header(chn->buf->p, idx, &ctx)) { + if (temp->len) + temp->str[temp->len++] = del; + memcpy(temp->str + temp->len, ctx.line, ctx.del); +@@ -10160,6 +10160,8 @@ smp_fetch_hdr_names(const struct arg *args, struct sample *smp, const char *kw, + static int + smp_fetch_hdr(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ /* possible keywords: req.hdr / hdr, res.hdr / shdr */ ++ struct channel *chn = ((kw[0] == 'h' || kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx *ctx = smp->ctx.a[0]; + const struct http_msg *msg; +@@ -10184,10 +10186,10 @@ smp_fetch_hdr(const struct arg *args, struct sample *smp, const char *kw, void * + occ = args[1].data.sint; + } + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + idx = &smp->strm->txn->hdr_idx; +- msg = ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) ? &smp->strm->txn->req : &smp->strm->txn->rsp; ++ msg = (!(chn->flags & CF_ISRESP) ? &smp->strm->txn->req : &smp->strm->txn->rsp); + + if (ctx && !(smp->flags & SMP_F_NOT_LAST)) + /* search for header from the beginning */ +@@ -10216,9 +10218,10 @@ smp_fetch_hdr(const struct arg *args, struct sample *smp, const char *kw, void * + static int + smp_fetch_hdr_cnt(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ /* possible keywords: req.hdr_cnt / hdr_cnt, res.hdr_cnt / shdr_cnt */ ++ struct channel *chn = ((kw[0] == 'h' || kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx ctx; +- const struct http_msg *msg; + int cnt; + const char *name = NULL; + int len = 0; +@@ -10228,14 +10231,13 @@ smp_fetch_hdr_cnt(const struct arg *args, struct sample *smp, const char *kw, vo + len = args->data.str.len; + } + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + idx = &smp->strm->txn->hdr_idx; +- msg = ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) ? &smp->strm->txn->req : &smp->strm->txn->rsp; + + ctx.idx = 0; + cnt = 0; +- while (http_find_header2(name, len, msg->chn->buf->p, idx, &ctx)) ++ while (http_find_header2(name, len, chn->buf->p, idx, &ctx)) + cnt++; + + smp->data.type = SMP_T_SINT; +@@ -10300,13 +10302,14 @@ smp_fetch_hdr_ip(const struct arg *args, struct sample *smp, const char *kw, voi + static int + smp_fetch_path(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + char *ptr, *end; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; +- end = txn->req.chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; ++ end = chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; + ptr = http_get_path(txn); + if (!ptr) + return 0; +@@ -10333,16 +10336,17 @@ smp_fetch_path(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_base(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + char *ptr, *end, *beg; + struct hdr_ctx ctx; + struct chunk *temp; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + ctx.idx = 0; +- if (!http_find_header2("Host", 4, txn->req.chn->buf->p, &txn->hdr_idx, &ctx) || !ctx.vlen) ++ if (!http_find_header2("Host", 4, chn->buf->p, &txn->hdr_idx, &ctx) || !ctx.vlen) + return smp_fetch_path(args, smp, kw, private); + + /* OK we have the header value in ctx.line+ctx.val for ctx.vlen bytes */ +@@ -10353,7 +10357,7 @@ smp_fetch_base(const struct arg *args, struct sample *smp, const char *kw, void + smp->data.u.str.len = ctx.vlen; + + /* now retrieve the path */ +- end = txn->req.chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; ++ end = chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; + beg = http_get_path(txn); + if (!beg) + beg = end; +@@ -10380,17 +10384,18 @@ smp_fetch_base(const struct arg *args, struct sample *smp, const char *kw, void + int + smp_fetch_base32(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + struct hdr_ctx ctx; + unsigned int hash = 0; + char *ptr, *beg, *end; + int len; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + ctx.idx = 0; +- if (http_find_header2("Host", 4, txn->req.chn->buf->p, &txn->hdr_idx, &ctx)) { ++ if (http_find_header2("Host", 4, chn->buf->p, &txn->hdr_idx, &ctx)) { + /* OK we have the header value in ctx.line+ctx.val for ctx.vlen bytes */ + ptr = ctx.line + ctx.val; + len = ctx.vlen; +@@ -10399,7 +10404,7 @@ smp_fetch_base32(const struct arg *args, struct sample *smp, const char *kw, voi + } + + /* now retrieve the path */ +- end = txn->req.chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; ++ end = chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; + beg = http_get_path(txn); + if (!beg) + beg = end; +@@ -10466,13 +10471,14 @@ smp_fetch_base32_src(const struct arg *args, struct sample *smp, const char *kw, + static int + smp_fetch_query(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + char *ptr, *end; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; +- ptr = txn->req.chn->buf->p + txn->req.sl.rq.u; ++ ptr = chn->buf->p + txn->req.sl.rq.u; + end = ptr + txn->req.sl.rq.u_l; + + /* look up the '?' */ +@@ -10491,11 +10497,13 @@ smp_fetch_query(const struct arg *args, struct sample *smp, const char *kw, void + static int + smp_fetch_proto_http(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); ++ + /* Note: hdr_idx.v cannot be NULL in this ACL because the ACL is tagged + * as a layer7 ACL, which involves automatic allocation of hdr_idx. + */ + +- CHECK_HTTP_MESSAGE_FIRST_PERM(); ++ CHECK_HTTP_MESSAGE_FIRST_PERM(chn); + + smp->data.type = SMP_T_BOOL; + smp->data.u.sint = 1; +@@ -10515,11 +10523,12 @@ smp_fetch_http_first_req(const struct arg *args, struct sample *smp, const char + static int + smp_fetch_http_auth(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + + if (!args || args->type != ARGT_USR) + return 0; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + if (!get_http_auth(smp->strm)) + return 0; +@@ -10534,10 +10543,12 @@ smp_fetch_http_auth(const struct arg *args, struct sample *smp, const char *kw, + static int + smp_fetch_http_auth_grp(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); ++ + if (!args || args->type != ARGT_USR) + return 0; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + if (!get_http_auth(smp->strm)) + return 0; +@@ -10828,10 +10839,10 @@ smp_fetch_capture_res_ver(const struct arg *args, struct sample *smp, const char + */ + int smp_fetch_cookie(const struct arg *args, struct sample *smp, const char *kw, void *private) + { +- struct http_txn *txn; ++ /* possible keywords: req.cookie / cookie / cook, res.cookie / scook / set-cookie */ ++ struct channel *chn = ((kw[0] == 'c' || kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx *ctx = smp->ctx.a[2]; +- const struct http_msg *msg; + const char *hdr_name; + int hdr_name_len; + char *sol; +@@ -10848,17 +10859,14 @@ int smp_fetch_cookie(const struct arg *args, struct sample *smp, const char *kw, + smp->ctx.a[2] = ctx; + } + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + +- txn = smp->strm->txn; + idx = &smp->strm->txn->hdr_idx; + +- if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) { +- msg = &txn->req; ++ if (!(chn->flags & CF_ISRESP)) { + hdr_name = "Cookie"; + hdr_name_len = 6; + } else { +- msg = &txn->rsp; + hdr_name = "Set-Cookie"; + hdr_name_len = 10; + } +@@ -10872,7 +10880,7 @@ int smp_fetch_cookie(const struct arg *args, struct sample *smp, const char *kw, + * next one. + */ + +- sol = msg->chn->buf->p; ++ sol = chn->buf->p; + if (!(smp->flags & SMP_F_NOT_LAST)) { + /* search for the header from the beginning, we must first initialize + * the search parameters. +@@ -10929,10 +10937,10 @@ int smp_fetch_cookie(const struct arg *args, struct sample *smp, const char *kw, + static int + smp_fetch_cookie_cnt(const struct arg *args, struct sample *smp, const char *kw, void *private) + { +- struct http_txn *txn; ++ /* possible keywords: req.cook_cnt / cook_cnt, res.cook_cnt / scook_cnt */ ++ struct channel *chn = ((kw[0] == 'c' || kw[2] == 'q') ? SMP_REQ_CHN(smp) : SMP_RES_CHN(smp)); + struct hdr_idx *idx; + struct hdr_ctx ctx; +- const struct http_msg *msg; + const char *hdr_name; + int hdr_name_len; + int cnt; +@@ -10942,22 +10950,19 @@ smp_fetch_cookie_cnt(const struct arg *args, struct sample *smp, const char *kw, + if (!args || args->type != ARGT_STR) + return 0; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + +- txn = smp->strm->txn; + idx = &smp->strm->txn->hdr_idx; + +- if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) { +- msg = &txn->req; ++ if (!(chn->flags & CF_ISRESP)) { + hdr_name = "Cookie"; + hdr_name_len = 6; + } else { +- msg = &txn->rsp; + hdr_name = "Set-Cookie"; + hdr_name_len = 10; + } + +- sol = msg->chn->buf->p; ++ sol = chn->buf->p; + val_end = val_beg = NULL; + ctx.idx = 0; + cnt = 0; +@@ -11287,6 +11292,7 @@ smp_fetch_param(char delim, const char *name, int name_len, const struct arg *ar + static int + smp_fetch_url_param(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_msg *msg; + char delim = '?'; + const char *name; +@@ -11308,16 +11314,16 @@ smp_fetch_url_param(const struct arg *args, struct sample *smp, const char *kw, + delim = *args[1].data.str.str; + + if (!smp->ctx.a[0]) { // first call, find the query string +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + msg = &smp->strm->txn->req; + +- smp->ctx.a[0] = find_param_list(msg->chn->buf->p + msg->sl.rq.u, ++ smp->ctx.a[0] = find_param_list(chn->buf->p + msg->sl.rq.u, + msg->sl.rq.u_l, delim); + if (!smp->ctx.a[0]) + return 0; + +- smp->ctx.a[1] = msg->chn->buf->p + msg->sl.rq.u + msg->sl.rq.u_l; ++ smp->ctx.a[1] = chn->buf->p + msg->sl.rq.u + msg->sl.rq.u_l; + + /* Assume that the context is filled with NULL pointer + * before the first call. +@@ -11339,6 +11345,7 @@ smp_fetch_url_param(const struct arg *args, struct sample *smp, const char *kw, + static int + smp_fetch_body_param(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_msg *msg; + unsigned long len; + unsigned long block1; +@@ -11357,19 +11364,15 @@ smp_fetch_body_param(const struct arg *args, struct sample *smp, const char *kw, + } + + if (!smp->ctx.a[0]) { // first call, find the query string +- CHECK_HTTP_MESSAGE_FIRST(); +- +- if ((smp->opt & SMP_OPT_DIR) == SMP_OPT_DIR_REQ) +- msg = &smp->strm->txn->req; +- else +- msg = &smp->strm->txn->rsp; ++ CHECK_HTTP_MESSAGE_FIRST(chn); + ++ msg = &smp->strm->txn->req; + len = http_body_bytes(msg); +- body = b_ptr(msg->chn->buf, -http_data_rewind(msg)); ++ body = b_ptr(chn->buf, -http_data_rewind(msg)); + + block1 = len; +- if (block1 > msg->chn->buf->data + msg->chn->buf->size - body) +- block1 = msg->chn->buf->data + msg->chn->buf->size - body; ++ if (block1 > chn->buf->data + chn->buf->size - body) ++ block1 = chn->buf->data + chn->buf->size - body; + + if (block1 == len) { + /* buffer is not wrapped (or empty) */ +@@ -11386,8 +11389,8 @@ smp_fetch_body_param(const struct arg *args, struct sample *smp, const char *kw, + /* buffer is wrapped, we need to defragment it */ + smp->ctx.a[0] = body; + smp->ctx.a[1] = body + block1; +- smp->ctx.a[2] = msg->chn->buf->data; +- smp->ctx.a[3] = msg->chn->buf->data + ( len - block1 ); ++ smp->ctx.a[2] = chn->buf->data; ++ smp->ctx.a[3] = chn->buf->data + ( len - block1 ); + } + } + return smp_fetch_param('&', name, name_len, args, smp, kw, private); +@@ -11422,17 +11425,18 @@ smp_fetch_url_param_val(const struct arg *args, struct sample *smp, const char * + static int + smp_fetch_url32(const struct arg *args, struct sample *smp, const char *kw, void *private) + { ++ struct channel *chn = SMP_REQ_CHN(smp); + struct http_txn *txn; + struct hdr_ctx ctx; + unsigned int hash = 0; + char *ptr, *beg, *end; + int len; + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST(chn); + + txn = smp->strm->txn; + ctx.idx = 0; +- if (http_find_header2("Host", 4, txn->req.chn->buf->p, &txn->hdr_idx, &ctx)) { ++ if (http_find_header2("Host", 4, chn->buf->p, &txn->hdr_idx, &ctx)) { + /* OK we have the header value in ctx.line+ctx.val for ctx.vlen bytes */ + ptr = ctx.line + ctx.val; + len = ctx.vlen; +@@ -11441,7 +11445,7 @@ smp_fetch_url32(const struct arg *args, struct sample *smp, const char *kw, void + } + + /* now retrieve the path */ +- end = txn->req.chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; ++ end = chn->buf->p + txn->req.sl.rq.u + txn->req.sl.rq.u_l; + beg = http_get_path(txn); + if (!beg) + beg = end; diff --git a/net/haproxy/patches/042-BUG-MEDIUM-maps-only-try-to-parse-the-default-value-when-its-present.patch b/net/haproxy/patches/042-BUG-MEDIUM-maps-only-try-to-parse-the-default-value-when-its-present.patch new file mode 100644 index 0000000000..9ba38fe8c5 --- /dev/null +++ b/net/haproxy/patches/042-BUG-MEDIUM-maps-only-try-to-parse-the-default-value-when-its-present.patch @@ -0,0 +1,38 @@ +commit 814ca94cbcba61a11485dedf80f6b35c34e4d74b +Author: Willy Tarreau +Date: Fri Apr 19 11:35:22 2019 +0200 + + BUG/MEDIUM: maps: only try to parse the default value when it's present + + Maps returning an IP address (e.g. map_str_ip) support an optional + default value which must be parsed. Unfortunately the parsing code does + not check for this argument's existence and uncondtionally tries to + resolve the argument whenever the output is of type address, resulting + in segfaults at parsing time when no such argument is provided. This + patch adds the appropriate check. + + This fix may be backported as far as 1.6. + + (cherry picked from commit aa5801bcaade82ce58b9a70f320b7d0389e444b0) + Signed-off-by: Willy Tarreau + (cherry picked from commit 0ad6d18945467f4d6defaad619ae49f939770ba2) + Signed-off-by: Christopher Faulet + +diff --git a/src/map.c b/src/map.c +index da399088..211d1911 100644 +--- a/src/map.c ++++ b/src/map.c +@@ -142,10 +142,10 @@ int sample_load_map(struct arg *arg, struct sample_conv *conv, + 1, err, file, line)) + return 0; + +- /* the maps of type IP have a string as defaultvalue. This +- * string canbe anipv4 or an ipv6, we must convert it. ++ /* the maps of type IP support a string as default value. This ++ * string can be an ipv4 or an ipv6, we must convert it. + */ +- if (desc->conv->out_type == SMP_T_ADDR) { ++ if (arg[1].type != ARGT_STOP && desc->conv->out_type == SMP_T_ADDR) { + struct sample_data data; + if (!map_parse_ip(arg[1].data.str.str, &data)) { + memprintf(err, "map: cannot parse default ip <%s>.", arg[1].data.str.str); diff --git a/net/haproxy/patches/043-BUG-MINOR-acl-properly-detect-pattern-type-SMP_T_ADDR.patch b/net/haproxy/patches/043-BUG-MINOR-acl-properly-detect-pattern-type-SMP_T_ADDR.patch new file mode 100644 index 0000000000..1c03f87a0b --- /dev/null +++ b/net/haproxy/patches/043-BUG-MINOR-acl-properly-detect-pattern-type-SMP_T_ADDR.patch @@ -0,0 +1,34 @@ +commit 4aa6348c04bc854b1dc47227b6931d43e704968d +Author: Willy Tarreau +Date: Fri Apr 19 11:45:20 2019 +0200 + + BUG/MINOR: acl: properly detect pattern type SMP_T_ADDR + + Since 1.6-dev4 with commit b2f8f087f ("MINOR: map: The map can return + IPv4 and IPv6"), maps can return both IPv4 and IPv6 addresses, which + is represented as SMP_T_ADDR at the output of the map converter. But + the ACL parser only checks for either SMP_T_IPV4 or SMP_T_IPV6 and + requires to see an explicit matching method specified. Given that it + uses the same pattern parser for both address families, it implicitly + is also compatible with SMP_T_ADDR, which ought to have been added + there. + + This fix should be backported as far as 1.6. + + (cherry picked from commit 78c5eec9497e1e60565492bc69581aea439e54cc) + Signed-off-by: Willy Tarreau + (cherry picked from commit ce727199a5b1a7c58cce1b0cfe79b91c6c138935) + Signed-off-by: Christopher Faulet + +diff --git a/src/acl.c b/src/acl.c +index f19b2d20..42339b43 100644 +--- a/src/acl.c ++++ b/src/acl.c +@@ -400,6 +400,7 @@ struct acl_expr *parse_acl_expr(const char **args, char **err, struct arg_list * + expr->pat.prune = pat_prune_fcts[PAT_MATCH_INT]; + expr->pat.expect_type = pat_match_types[PAT_MATCH_INT]; + break; ++ case SMP_T_ADDR: + case SMP_T_IPV4: + case SMP_T_IPV6: + expr->pat.parse = pat_parse_fcts[PAT_MATCH_IP]; diff --git a/net/haproxy/patches/044-BUG-MEDIUM-thread-http-Add-missing-locks-in-set-map-and-add-acl-HTTP-rules.patch b/net/haproxy/patches/044-BUG-MEDIUM-thread-http-Add-missing-locks-in-set-map-and-add-acl-HTTP-rules.patch new file mode 100644 index 0000000000..55ccfa6c25 --- /dev/null +++ b/net/haproxy/patches/044-BUG-MEDIUM-thread-http-Add-missing-locks-in-set-map-and-add-acl-HTTP-rules.patch @@ -0,0 +1,48 @@ +commit 8276ea30400887cb25186571ac62252da97b91df +Author: Christopher Faulet +Date: Fri Apr 19 14:50:55 2019 +0200 + + BUG/MEDIUM: thread/http: Add missing locks in set-map and add-acl HTTP rules + + Locks are missing in the rules "http-request set-map" and "http-response + add-acl" when an acl or map update is performed. Pattern elements must be + locked. + + This patch must be backported to 1.9 and 1.8. For the 1.8, the HTX part must be + ignored. + + (cherry picked from commit e84289e5854aa3b00cd19032387f435ca6748491) + Signed-off-by: Christopher Faulet + (cherry picked from commit 82dedc4add923bd1ff1b47a559a23e83886521a8) + Signed-off-by: Christopher Faulet + +diff --git a/src/proto_http.c b/src/proto_http.c +index 9beaa137..ccacd6a4 100644 +--- a/src/proto_http.c ++++ b/src/proto_http.c +@@ -2717,12 +2717,14 @@ resume_execution: + value->str[value->len] = '\0'; + + /* perform update */ ++ HA_SPIN_LOCK(PATREF_LOCK, &ref->lock); + if (pat_ref_find_elt(ref, key->str) != NULL) + /* update entry if it exists */ + pat_ref_set(ref, key->str, value->str, NULL); + else + /* insert a new entry */ + pat_ref_add(ref, key->str, value->str, NULL); ++ HA_SPIN_UNLOCK(PATREF_LOCK, &ref->lock); + + free_trash_chunk(key); + free_trash_chunk(value); +@@ -2978,8 +2980,10 @@ resume_execution: + + /* perform update */ + /* check if the entry already exists */ ++ HA_SPIN_LOCK(PATREF_LOCK, &ref->lock); + if (pat_ref_find_elt(ref, key->str) == NULL) + pat_ref_add(ref, key->str, NULL, NULL); ++ HA_SPIN_UNLOCK(PATREF_LOCK, &ref->lock); + + free_trash_chunk(key); + break; diff --git a/net/haproxy/patches/045-BUG-MINOR-51d-Get-the-request-channel-to-call-CHECK_HTTP_MESSAGE_FIRST.patch b/net/haproxy/patches/045-BUG-MINOR-51d-Get-the-request-channel-to-call-CHECK_HTTP_MESSAGE_FIRST.patch new file mode 100644 index 0000000000..85d55548c4 --- /dev/null +++ b/net/haproxy/patches/045-BUG-MINOR-51d-Get-the-request-channel-to-call-CHECK_HTTP_MESSAGE_FIRST.patch @@ -0,0 +1,30 @@ +commit 02fd3cd55a2232703494ad5c317907aba21783fe +Author: Christopher Faulet +Date: Fri Apr 19 15:22:29 2019 +0200 + + BUG/MINOR: 51d: Get the request channel to call CHECK_HTTP_MESSAGE_FIRST() + + Since the commit 89dc49935 ("BUG/MAJOR: http_fetch: Get the channel depending on + the keyword used"), the right channel must be passed as argument when the macro + CHECK_HTTP_MESSAGE_FIRST is called. + + This patch must be backported to 1.9. + + (cherry picked from commit 2db9dac4c81bbb97b45c9f0ff73ed9fe24a2ba83) + Signed-off-by: Christopher Faulet + (cherry picked from commit 4fc4c6a9d5effccab3f63909cc86bca452f1be1e) + Signed-off-by: Christopher Faulet + +diff --git a/src/51d.c b/src/51d.c +index 03101136..a0b683b1 100644 +--- a/src/51d.c ++++ b/src/51d.c +@@ -384,7 +384,7 @@ static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw + * Data type has to be reset to ensure the string output is processed + * correctly. + */ +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST((smp->strm ? &smp->strm->req : NULL)); + smp->data.type = SMP_T_STR; + + /* Flags the sample to show it uses constant memory*/ diff --git a/net/haproxy/patches/046-BUG-MINOR-da-Get-the-request-channel-to-call-CHECK_HTTP_MESSAGE_FIRST.patch b/net/haproxy/patches/046-BUG-MINOR-da-Get-the-request-channel-to-call-CHECK_HTTP_MESSAGE_FIRST.patch new file mode 100644 index 0000000000..a245ed0036 --- /dev/null +++ b/net/haproxy/patches/046-BUG-MINOR-da-Get-the-request-channel-to-call-CHECK_HTTP_MESSAGE_FIRST.patch @@ -0,0 +1,30 @@ +commit 01d9157013729859fdb7470887d78d67a3cdf6b9 +Author: Christopher Faulet +Date: Fri Apr 19 15:26:01 2019 +0200 + + BUG/MINOR: da: Get the request channel to call CHECK_HTTP_MESSAGE_FIRST() + + Since the commit 89dc49935 ("BUG/MAJOR: http_fetch: Get the channel depending on + the keyword used"), the right channel must be passed as argument when the macro + CHECK_HTTP_MESSAGE_FIRST is called. + + This patch must be backported to 1.9. + + (cherry picked from commit f48552f2c10a2f956d7bd1eb02a6d694d2b5c5d3) + Signed-off-by: Christopher Faulet + (cherry picked from commit 2a38fa09b588d8b30fabc77282e66ef613336ee7) + Signed-off-by: Christopher Faulet + +diff --git a/src/da.c b/src/da.c +index 685a79d1..cd4050bf 100644 +--- a/src/da.c ++++ b/src/da.c +@@ -293,7 +293,7 @@ static int da_haproxy_fetch(const struct arg *args, struct sample *smp, const ch + return 1; + } + +- CHECK_HTTP_MESSAGE_FIRST(); ++ CHECK_HTTP_MESSAGE_FIRST((smp->strm ? &smp->strm->req : NULL)); + smp->data.type = SMP_T_STR; + + /** diff --git a/net/haproxy/patches/047-BUG-MINOR-spoe-Dont-systematically-wakeup-SPOE-stream-in-the-applet-handler.patch b/net/haproxy/patches/047-BUG-MINOR-spoe-Dont-systematically-wakeup-SPOE-stream-in-the-applet-handler.patch new file mode 100644 index 0000000000..3165d8a7e4 --- /dev/null +++ b/net/haproxy/patches/047-BUG-MINOR-spoe-Dont-systematically-wakeup-SPOE-stream-in-the-applet-handler.patch @@ -0,0 +1,29 @@ +commit 1526ce4e6f5fb241ca236bd2ac870cdb30e054fd +Author: Christopher Faulet +Date: Tue Apr 23 15:39:32 2019 +0200 + + BUG/MINOR: spoe: Don't systematically wakeup SPOE stream in the applet handler + + This can lead to wakeups in loop between the SPOE stream and the SPOE applets + waiting to receive agent messages (mainly AGENT-HELLO and AGENT-DISCONNECT). + + This patch must be backported to 1.9 and 1.8. + + (cherry picked from commit 371723b0c2a2e38ae14e1e6f6a7581ef3e2491cf) + Signed-off-by: Christopher Faulet + (cherry picked from commit fe0ccea6bb93406ca0a7339cdf17357b1a283e59) + Signed-off-by: Christopher Faulet + +diff --git a/src/flt_spoe.c b/src/flt_spoe.c +index 95f30898..f6109778 100644 +--- a/src/flt_spoe.c ++++ b/src/flt_spoe.c +@@ -1929,8 +1929,6 @@ spoe_handle_appctx(struct appctx *appctx) + + if (SPOE_APPCTX(appctx)->task->expire != TICK_ETERNITY) + task_queue(SPOE_APPCTX(appctx)->task); +- si_oc(si)->flags |= CF_READ_DONTWAIT; +- task_wakeup(si_strm(si)->task, TASK_WOKEN_IO); + } + + struct applet spoe_applet = { diff --git a/net/haproxy/patches/008-deprecated-openssl.patch b/net/haproxy/patches/048-deprecated-openssl.patch similarity index 100% rename from net/haproxy/patches/008-deprecated-openssl.patch rename to net/haproxy/patches/048-deprecated-openssl.patch