From 417e24e22b012be3faecbfda0e8bd48ac770df9d Mon Sep 17 00:00:00 2001 From: Thibaut Artis Date: Wed, 11 Sep 2024 18:02:14 +0200 Subject: [PATCH] jail: Try disabling Transparent Hugepage on Linux Disabling Transparent Hugepage has often been the solution to solve hard-to-diagnose instability issues and despite improvements in this area compared to the RHEL6 era, our recommandation is still to avoid THP to this day. In addition to refreshing the documentation on this topic, the Linux jail will tentatively disable THP or warn when it cannot. --- bin/varnishd/mgt/mgt_jail_linux.c | 71 ++++++++++++++++++++--- doc/sphinx/installation/platformnotes.rst | 30 ++++++---- doc/sphinx/reference/varnishd.rst | 1 + 3 files changed, 84 insertions(+), 18 deletions(-) diff --git a/bin/varnishd/mgt/mgt_jail_linux.c b/bin/varnishd/mgt/mgt_jail_linux.c index 93e5c2a6d16..9e6c41c30f0 100644 --- a/bin/varnishd/mgt/mgt_jail_linux.c +++ b/bin/varnishd/mgt/mgt_jail_linux.c @@ -48,11 +48,61 @@ #include "mgt/mgt.h" #include "common/heritage.h" +static int +vjl_set_thp(const char *arg) +{ + int val; + + if (!strcmp(arg, "ignore")) + return (0); + if (!strcmp(arg, "enable")) + val = 0; + else if (!strcmp(arg, "disable")) + val = 1; + else { + ARGV_ERR( + "linux jail: unknown value '%s' for argument transparent_hugepage.\n", + arg); + } + if (prctl(PR_SET_THP_DISABLE, val, 0, 0, 0) != 0) { + MGT_Complain(C_ERR, + "Could not %s Transparent Hugepage: %s (%d)", + arg, VAS_errtxt(errno), errno); + } + return (0); +} + static int vjl_init(char **args) { + char **unix_args; + int ret = 0; + size_t i; + + if (args == NULL) { + /* Autoconfig */ + if (vjl_set_thp("disable") != 0) + return (1); + return (jail_tech_unix.init(NULL)); + } - return jail_tech_unix.init(args); + for (i = 0; args[i] != NULL; i++); + unix_args = calloc(i + 1, sizeof *unix_args); + AN(unix_args); + + for (i = 0; *args != NULL && ret == 0; args++) { + if (!strncmp(*args, "transparent_hugepage=", 22)) { + ret = vjl_set_thp((*args) + 22); + } else { + unix_args[i] = *args; + i++; + } + } + + if (ret == 0) + ret = jail_tech_unix.init(unix_args); + free(unix_args); + return (ret); } static void @@ -94,17 +144,24 @@ vjl_make_workdir(const char *dname, const char *what, struct vsb *vsb) vjl_master(JAIL_MASTER_FILE); if (statfs(dname, &info) != 0) { - if (vsb) - VSB_printf(vsb, "Could not stat working directory '%s': %s (%d)\n", dname, VAS_errtxt(errno), errno); - else - MGT_Complain(C_ERR, "Could not stat working directory '%s': %s (%d)", dname, VAS_errtxt(errno), errno); + if (vsb) { + VSB_printf(vsb, + "Could not stat working directory '%s': %s (%d)\n", + dname, VAS_errtxt(errno), errno); + } else { + MGT_Complain(C_ERR, + "Could not stat working directory '%s': %s (%d)", + dname, VAS_errtxt(errno), errno); + } return (1); } if (info.f_type != TMPFS_MAGIC) { if (vsb != NULL) - VSB_printf(vsb, "Working directory not mounted on tmpfs partition\n"); + VSB_printf(vsb, + "Working directory not mounted on tmpfs partition\n"); else - MGT_Complain(C_INFO, "Working directory not mounted on tmpfs partition"); + MGT_Complain(C_INFO, + "Working directory not mounted on tmpfs partition"); } vjl_master(JAIL_MASTER_LOW); return (0); diff --git a/doc/sphinx/installation/platformnotes.rst b/doc/sphinx/installation/platformnotes.rst index 494e4eb9d6d..ab4ad61280e 100644 --- a/doc/sphinx/installation/platformnotes.rst +++ b/doc/sphinx/installation/platformnotes.rst @@ -11,21 +11,29 @@ On some platforms it is necessary to adjust the operating system before running Varnish on it. The systems and steps known to us are described in this section. -Transparent hugepages on Redhat Enterprise Linux 6 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Transparent Hugepage on Linux +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -On RHEL6 Transparent Hugepage kernel support is enabled by default. -This is known to cause sporadic crashes of Varnish. +On certain Linux distributions Transparent Hugepage kernel support is enabled +by default. This is known to cause instability of Varnish. -It is recommended to disable transparent hugepages on affected -systems. This can be done with -``echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled`` -(runtime) or by adding "transparent_hugepage=never" to the kernel boot -line in the "/etc/grub.conf" file (persistent). +It is recommended to disable Transparent Hugepage on affected systems. +If Varnish is the only significant service running on this system, this can be +done during runtime with:: -On Debian/Ubuntu systems running 3.2 kernels the default value is "madvise" and -does not need to be changed. + echo never > /sys/kernel/mm/transparent_hugepage/enabled +Alternatively, this can be persisted in your bootloader configuration by adding +``transparent_hugepage=never`` to the kernel command line. + +On other systems the default value is ``madvise`` and does not need to be +changed. Either way, the Linux :ref:`ref-varnishd-opt_j` will try to disable +Transparent Hugepage in the ``varnishd`` process. + +The general recommendation is to mount the working directory in a ``tmpfs`` +partition, usually ``/var/lib/varnish``. By default tmpfs should be mounted with +Transparent Hugepage disabled. Consider mounting the working directory with the +``huge=never`` mount option if that is not the case with your OS vendor. OpenVZ ~~~~~~ diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index a3c445d142d..148af405cad 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -461,6 +461,7 @@ specific options. Available jails are: - It warns when *workdir* is not on a ``tmpfs``. - It tries to keep the process dumpable after dropping privileges. + - It tries to disable Transparent Hugepage in ``varnishd`` -j