From cfbebe559c437713035b486152fbf25e4dc35dd9 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 11 May 2015 12:21:55 +0200 Subject: [PATCH] memory: Force the kernel to enable working HugeTLB allocation If huge page allocation fails then do one of two things: - Increase sysctl kernel.shmmax if the maximum shared memory allocation size is less than the size of one huge page. (That is a misconfigured kernel from our perspective.) Fixes #366. - Increase sysctl vm.nr_hugepages otherwise to have the kernel reserve a new huge page that we can allocate. (Try this up to three times per allocation, as before.) To change system-wide sysctl values is a fairly presumptuious thing to do. For this reason we now print clear messages about what we are doing: [memory: Enabling huge pages for shm: sysctl kernel.shmmax 1000000 -> 2097152] [memory: Provisioned a huge page: sysctl vm.nr_hugepages 0 -> 1] [memory: Provisioned a huge page: sysctl vm.nr_hugepages 1 -> 2] [memory: Provisioned a huge page: sysctl vm.nr_hugepages 2 -> 3] [memory: Provisioned a huge page: sysctl vm.nr_hugepages 3 -> 4] [memory: Provisioned a huge page: sysctl vm.nr_hugepages 4 -> 5] On balance I feel that it is reasonable that we ask for forgiveness rather than permission on these operations. This is all done with ljsyscall instead of procfs. This is much nicer! --- src/core/memory.lua | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/core/memory.lua b/src/core/memory.lua index 2d3067fbf4..fd454927af 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -8,6 +8,7 @@ module(...,package.seeall) local ffi = require("ffi") local C = ffi.C +local syscall = require("syscall") local lib = require("core.lib") require("core.memory_h") @@ -61,16 +62,21 @@ function allocate_hugetlb_chunk () end function reserve_new_page () + -- Check that we have permission lib.root_check("error: must run as root to allocate memory for DMA") - set_hugepages(get_hugepages() + 1) -end - -function get_hugepages () - return lib.readfile("/proc/sys/vm/nr_hugepages", "*n") -end - -function set_hugepages (n) - lib.writefile("/proc/sys/vm/nr_hugepages", tostring(n)) + -- Is the kernel shm limit too low for huge pages? + if huge_page_size > tonumber(syscall.sysctl("kernel.shmmax")) then + -- Yes: fix that + local old = syscall.sysctl("kernel.shmmax", tostring(huge_page_size)) + io.write("[memory: Enabling huge pages for shm: ", + "sysctl kernel.shmmax ", old, " -> ", huge_page_size, "]\n") + else + -- No: try provisioning an additional page + local have = tonumber(syscall.sysctl("vm.nr_hugepages")) + local want = have + 1 + syscall.sysctl("vm.nr_hugepages", tostring(want)) + io.write("[memory: Provisioned a huge page: sysctl vm.nr_hugepages ", have, " -> ", want, "]\n") + end end function get_huge_page_size ()