Merge pull request #433 from lukego/dynahz

core: Count "frees per second" (fps) and do dynamic idling

src/core/app.lua

@@ -23,10 +23,32 @@ link_table, link_array = {}, {}
 
 configuration = config.new()
 
--- Count of the number of breaths taken
-breaths = 0
--- Ideal number of breaths per second
+-- Counters for statistics.
+-- TODO: Move these over to the counters framework
+breaths = 0			-- Total breaths taken
+frees   = 0			-- Total packets freed
+freebits = 0			-- Total packet bits freed (for 10GbE)
+freebytes = 0			-- Total packet bytes freed
+
+-- Breathing regluation to reduce CPU usage when idle by calling usleep(3).
+--
+-- There are two modes available:
+--
+--   Hz = <n> means to aim for an exact <n> breaths per second rhythm
+--   Hz = false means dynamic adjustment of the breathing interval
+--
+-- Dynamic adjustment automatically scales the time to sleep between
+-- breaths from nothing up to maxsleep (default: 100us). If packets
+-- are processed during a breath then the sleep period is halved, and
+-- if no packets are processed during a breath then the sleep interval
+-- is increased by one microsecond.
+--
+-- The default is dynamic adjustment which should work well for the
+-- majority of cases.
+
 Hz = false
+sleep = 0
+maxsleep = 100
 
 -- Return current monotonic time in seconds.
 -- Can be used to drive timers in apps.
@@ -211,6 +233,9 @@ function main (options)
 end
 
 local nextbreath
+local lastfrees = 0
+local lastfreebits = 0
+local lastfreebytes = 0
 -- Wait between breaths to keep frequency with Hz.
 function pace_breathing ()
    if Hz then
@@ -221,6 +246,16 @@ function pace_breathing ()
          monotonic_now = C.get_monotonic_time()
       end
       nextbreath = math.max(nextbreath + 1/Hz, monotonic_now)
+   else
+      if lastfrees == frees then
+	 sleep = math.min(sleep + 1, maxsleep)
+	 C.usleep(sleep)
+      else
+	 sleep = math.floor(sleep/2)
+      end
+      lastfrees = frees
+      lastfreebytes = freebytes
+      lastfreebits = freebits
    end
 end
 
@@ -263,32 +298,77 @@ function breathe ()
 end
 
 function report (options)
+   if not options or options.showload then
+      report_load()
+   end
+   if options and options.showlinks then
+      report_links()
+   end
+   if options and options.showapps then
+      report_apps()
+   end
+end
+
+-- Load reporting prints several metrics:
+--   time - period of time that the metrics were collected over
+--   fps  - frees per second (how many calls to packet.free())
+--   fpb  - frees per breath
+--   bpp  - bytes per packet (average packet size)
+local lastloadreport = nil
+local reportedfrees = nil
+local reportedbreaths = nil
+function report_load ()
+   if lastloadreport then
+      local interval = now() - lastloadreport
+      local newfrees   = frees - reportedfrees
+      local newbytes   = freebytes - reportedfreebytes
+      local newbits    = freebits - reportedfreebits
+      local newbreaths = breaths - reportedbreaths
+      local fps = math.floor(newfrees/interval)
+      local fbps = math.floor(newbits/interval)
+      local fpb = math.floor(newfrees/newbreaths)
+      local bpp = math.floor(newbytes/newfrees)
+      print(("load: time: %-2.2fs  fps: %-9s fpGbps: %-3.3f fpb: %-3s bpp: %-4s sleep: %-4dus"):format(
+	    interval,
+	    lib.comma_value(fps),
+	    fbps / 1e9,
+	    lib.comma_value(fpb),
+	    (bpp ~= bpp) and "-" or tostring(bpp), -- handle NaN
+	    sleep))
+   end
+   lastloadreport = now()
+   reportedfrees = frees
+   reportedfreebits = freebits
+   reportedfreebytes = freebytes
+   reportedbreaths = breaths
+end
+
+function report_links ()
+   print("link report:")
    local function loss_rate(drop, sent)
       sent = tonumber(sent)
       if not sent or sent == 0 then return 0 end
       return tonumber(drop) * 100 / (tonumber(drop)+sent)
    end
-   if not options or options.showlinks then
-      print("link report:")
-      local names = {}
-      for name in pairs(link_table) do table.insert(names, name) end
-      table.sort(names)
-      for i, name in ipairs(names) do
-	 l = link_table[name]
-         print(("%20s sent on %s (loss rate: %d%%)"):format(
-		  lib.comma_value(l.stats.txpackets),
-		  name, loss_rate(l.stats.txdrop, l.stats.txpackets)))
-      end
+   local names = {}
+   for name in pairs(link_table) do table.insert(names, name) end
+   table.sort(names)
+   for i, name in ipairs(names) do
+      l = link_table[name]
+      print(("%20s sent on %s (loss rate: %d%%)"):format(
+	    lib.comma_value(l.stats.txpackets),
+	    name, loss_rate(l.stats.txdrop, l.stats.txpackets)))
    end
-   if options and options.showapps then
-      print ("apps report")
-      for name, app in pairs(app_table) do
-         if app.dead then
-            print (name, ("[dead: %s]"):format(app.dead.error))
-         elseif app.report then
-            print (name)
-            with_restart(app, app.report)
-         end
+end
+
+function report_apps ()
+   print ("apps report:")
+   for name, app in pairs(app_table) do
+      if app.dead then
+	 print(name, ("[dead: %s]"):format(app.dead.error))
+      elseif app.report then
+	 print(name)
+	 with_restart(app, app.report)
       end
    end
 end

src/core/lib.lua

@@ -238,6 +238,7 @@ function hexundump(h, n)
 end
 
 function comma_value(n) -- credit http://richard.warburton.it
+   if n ~= n then return "NaN" end
    local left,num,right = string.match(n,'^([^%d]*%d)(%d*)(.-)$')
    return left..(num:reverse():gsub('(%d%d%d)','%1,'):reverse())..right
 end

src/core/packet.lua

@@ -76,10 +76,18 @@ function from_pointer (ptr, len) return append(allocate(), ptr, len) end
 function from_string (d)         return from_pointer(d, #d) end
 
 -- Free a packet that is no longer in use.
-function free (p)
-   --ffi.fill(p, header_size, 0)
+local function free_internal (p)
    p.length = 0
    freelist_add(packets_fl, p)
+end   
+
+function free (p)
+   engine.frees = engine.frees + 1
+   engine.freebytes = engine.freebytes + p.length
+   -- Calculate bits of physical capacity required for packet on 10GbE
+   -- Account for minimum data size and overhead of CRC and inter-packet gap
+   engine.freebits = engine.freebits + (math.max(p.length, 46) + 4 + 5) * 8
+   free_internal(p)
 end
 
 -- Return pointer to packet data.
@@ -94,7 +102,7 @@ function preallocate_step()
    end
 
    for i=1, packet_allocation_step do
-      free(new_packet())
+      free_internal(new_packet(), true)
    end
    packets_allocated = packets_allocated + packet_allocation_step
    packet_allocation_step = 2 * packet_allocation_step

src/program/snabbnfv/nfvconfig.lua

@@ -29,8 +29,8 @@ function load (file, pciaddr, sockpath)
    for _,t in ipairs(ports) do
       local vlan, mac_address = t.vlan, t.mac_address
       local name = port_name(t)
-      local NIC = "NIC_"..name
-      local Virtio = "Virtio_"..name
+      local NIC = name.."_NIC"
+      local Virtio = name.."_Virtio"
       config.app(c, NIC, require(device_info.driver).driver,
                  {pciaddr = pciaddr,
                   vmdq = true,
@@ -39,26 +39,26 @@ function load (file, pciaddr, sockpath)
       config.app(c, Virtio, VhostUser, {socket_path=sockpath:format(t.port_id)})
       local VM_rx, VM_tx = Virtio..".rx", Virtio..".tx"
       if t.tx_police_gbps then
-         local TxLimit = "TxLimit_"..name
+         local TxLimit = name.."_TxLimit"
          local rate = t.tx_police_gbps * 1e9 / 8
          config.app(c, TxLimit, RateLimiter, {rate = rate, bucket_capacity = rate})
          config.link(c, VM_tx.." -> "..TxLimit..".input")
          VM_tx = TxLimit..".output"
       end
       if t.ingress_filter then
-         local Filter = "Filter_in_"..name
+         local Filter = name.."_Filter_in"
          config.app(c, Filter, PacketFilter, t.ingress_filter)
          config.link(c, Filter..".tx -> " .. VM_rx)
          VM_rx = Filter..".rx"
       end
       if t.egress_filter then
-         local Filter = 'Filter_out_'..name
+         local Filter = name..'_Filter_out'
          config.app(c, Filter, PacketFilter, t.egress_filter)
          config.link(c, VM_tx..' -> '..Filter..'.rx')
          VM_tx = Filter..'.tx'
       end
       if t.tunnel and t.tunnel.type == "L2TPv3" then
-         local Tunnel = "Tunnel_"..name
+         local Tunnel = name.."_Tunnel"
          local conf = {local_address = t.tunnel.local_ip,
                        remote_address = t.tunnel.remote_ip,
                        local_cookie = t.tunnel.local_cookie,
@@ -67,7 +67,7 @@ function load (file, pciaddr, sockpath)
          config.app(c, Tunnel, L2TPv3, conf)
          -- Setup IPv6 neighbor discovery/solicitation responder.
          -- This will talk to our local gateway.
-         local ND = "ND_"..name
+         local ND = name.."_ND"
          config.app(c, ND, nd_light,
                     {local_mac = mac_address,
                      local_ip = t.tunnel.local_ip,
@@ -81,7 +81,7 @@ function load (file, pciaddr, sockpath)
          VM_rx, VM_tx = ND..".south", ND..".south"
       end
       if t.rx_police_gbps then
-         local RxLimit = "RxLimit_"..name
+         local RxLimit = name.."_RxLimit"
          local rate = t.rx_police_gbps * 1e9 / 8
          config.app(c, RxLimit, RateLimiter, {rate = rate, bucket_capacity = rate})
          config.link(c, RxLimit..".output -> "..VM_rx)

src/program/snabbnfv/traffic/README

@@ -1,5 +1,11 @@
 snabbnfv traffic [OPTIONS] <pci-address> <config-file> <socket-path>
 
+  -k SECONDS, --link-report-interval SECONDS
+                             Print an link status report every SECONDS.
+			     Default: 60s
+  -l SECONDS, --load-report-interval SECONDS
+                             Print a processing load report every SECONDS.
+			     Default: 1s
   -B NPACKETS, --benchmark NPACKETS
                              Benchmark processing NPACKETS.
   -h, --help

src/program/snabbnfv/traffic/traffic.lua

@@ -5,22 +5,37 @@ local nfvconfig = require("program.snabbnfv.nfvconfig")
 local usage = require("program.snabbnfv.traffic.README_inc")
 local ffi = require("ffi")
 local C = ffi.C
+local timer = require("core.timer")
 
 local long_opts = {
    benchmark     = "B",
    help          = "h",
+   ["link-report-interval"] = "k",
+   ["load-report-interval"] = "l",
    ["long-help"] = "H"
 }
 
 function run (args)
    local opt = {}
    local benchpackets
+   local linkreportinterval = 60
+   local loadreportinterval = 1
    function opt.B (arg) benchpackets = tonumber(arg)      end
    function opt.h (arg) print(short_usage()) main.exit(1) end
    function opt.H (arg) print(long_usage())  main.exit(1) end
-   args = lib.dogetopt(args, opt, "hHB:", long_opts)
+   function opt.k (arg) linkreportinterval = tonumber(arg) end
+   function opt.l (arg) loadreportinterval = tonumber(arg) end
+   args = lib.dogetopt(args, opt, "hHB:k:l:", long_opts)
    if #args == 3 then
       local pciaddr, confpath, sockpath = unpack(args)
+      if loadreportinterval > 0 then
+	 local t = timer.new("nfvloadreport", engine.report_load, loadreportinterval*1e9, 'repeating')
+	 timer.activate(t)
+      end
+      if linkreportinterval > 0 then
+	 local t = timer.new("nfvlinkreport", engine.report_links, linkreportinterval*1e9, 'repeating')
+	 timer.activate(t)
+      end
       if benchpackets then
 	 print("snabbnfv traffic starting (benchmark mode)")
 	 bench(pciaddr, confpath, sockpath, benchpackets)
@@ -44,27 +59,23 @@ function traffic (pciaddr, confpath, sockpath)
    engine.log = true
    local mtime = 0
    while true do
-      for i = 1, 60 do
-         local mtime2 = C.stat_mtime(confpath)
-         if mtime2 ~= mtime then
-            print("Loading " .. confpath)
-            engine.configure(nfvconfig.load(confpath, pciaddr, sockpath))
-            mtime = mtime2
-         end
-         engine.main({duration=1})
-         -- Flush buffered log messages every 1s
-         io.flush()
+      local mtime2 = C.stat_mtime(confpath)
+      if mtime2 ~= mtime then
+	 print("Loading " .. confpath)
+	 engine.configure(nfvconfig.load(confpath, pciaddr, sockpath))
+	 mtime = mtime2
       end
-      -- Report each minute
-      engine.report()
+      engine.main({duration=1, no_report=true})
+      -- Flush buffered log messages every 1s
+      io.flush()
    end
 end
 
 -- Run in benchmark mode.
 function bench (pciaddr, confpath, sockpath, npackets)
    npackets = tonumber(npackets)
    local ports = dofile(confpath)
-   local nic = "NIC_"..(nfvconfig.port_name(ports[1]))
+   local nic = (nfvconfig.port_name(ports[1])).."_NIC"
    engine.log = true
    engine.Hz = false