Merge pull request #424 from lukego/integrate

Integration branch merge containing PRs to master.

Makefile

@@ -6,7 +6,7 @@ COBJ   = $(CSRC:.c=.o)
 LUAJIT_O := deps/luajit/src/libluajit.a
 SYSCALL  := src/syscall.lua
 
-LUAJIT_CFLAGS := -DLUAJIT_USE_PERFTOOLS -DLUAJIT_USE_GDBJIT -DLUAJIT_NUMMODE=3 -include $(CURDIR)/gcc-preinclude.h
+LUAJIT_CFLAGS := -include $(CURDIR)/gcc-preinclude.h
 
 all: $(LUAJIT_O) $(SYSCALL)
 	@echo "Building snabbswitch"

src/core/app.lua

@@ -26,7 +26,7 @@ configuration = config.new()
 -- Count of the number of breaths taken
 breaths = 0
 -- Ideal number of breaths per second
-Hz = 10000
+Hz = false
 
 -- Return current monotonic time in seconds.
 -- Can be used to drive timers in apps.
@@ -268,10 +268,15 @@ function report (options)
       return tonumber(drop) * 100 / (tonumber(drop)+sent)
    end
    if not options or options.showlinks then
-      print("link report")
-      for name, l in pairs(link_table) do
-         print(("%s sent on %s (loss rate: %d%%))"):format(l.stats.txpackets,
-            name, loss_rate(l.stats.txdrop, l.stats.txpackets)))
+      print("link report:")
+      local names = {}
+      for name in pairs(link_table) do table.insert(names, name) end
+      table.sort(names)
+      for i, name in ipairs(names) do
+	 l = link_table[name]
+         print(("%20s sent on %s (loss rate: %d%%)"):format(
+		  lib.comma_value(l.stats.txpackets),
+		  name, loss_rate(l.stats.txdrop, l.stats.txpackets)))
       end
    end
    if options and options.showapps then

src/core/clib.h

@@ -61,6 +61,3 @@ uint16_t ntohs(uint16_t);
 uint32_t htonl(uint32_t);
 uint32_t ntohl(uint32_t);
 
-// geteuid(2) - get effective user identity
-int geteuid();
-

src/core/lib.lua

@@ -3,6 +3,7 @@ module(...,package.seeall)
 local ffi = require("ffi")
 local C = ffi.C
 local getopt = require("lib.lua.alt_getopt")
+local syscall = require("syscall")
 require("core.clib_h")
 
 -- Returns true if x and y are structurally similar (isomorphic).
@@ -419,6 +420,14 @@ function have_module (name)
    end
 end
 
+-- Exit with an error if we are not running as root.
+function root_check (message)
+   if syscall.geteuid() ~= 0 then
+      print(message or "error: must run as root")
+      main.exit(1)
+   end
+end
+
 function selftest ()
    print("selftest: lib")
    print("Testing equal")

src/core/main.lua

@@ -81,10 +81,6 @@ function initialize ()
    require("core.lib")
    require("core.clib_h")
    require("core.lib_h")
-   if C.geteuid() ~= 0 then
-      print("error: snabb has to run as root.")
-      os.exit(1)
-   end
    -- Global API
    _G.config = require("core.config")
    _G.engine = require("core.app")

src/core/memory.c

@@ -86,11 +86,13 @@ void *allocate_huge_page(int size)
   shmid = shmget(IPC_PRIVATE, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
   tmpptr = shmat(shmid, NULL, 0);
   if (tmpptr == MAP_FAILED) { goto fail; }
+  if (mlock(tmpptr, size) != 0) { goto fail; }
   physical_address = virtual_to_physical(tmpptr);
   if (physical_address == 0) { goto fail; }
   virtual_address = physical_address | 0x500000000000ULL;
   realptr = shmat(shmid, (void*)virtual_address, 0);
   if (realptr == MAP_FAILED) { goto fail; }
+  if (mlock(realptr, size) != 0) { goto fail; }
   memset(realptr, 0, size); // zero memory to avoid potential surprises
   shmdt(tmpptr);
   shmctl(shmid, IPC_RMID, 0);
@@ -102,9 +104,3 @@ void *allocate_huge_page(int size)
   return NULL;
 }
 
-// Lock all current and future virtual memory in a stable physical location.
-int lock_memory()
-{
-  return mlockall(MCL_CURRENT | MCL_FUTURE);
-}
-

src/core/memory.lua

@@ -61,6 +61,7 @@ function allocate_hugetlb_chunk ()
 end
 
 function reserve_new_page ()
+   lib.root_check("error: must run as root to allocate memory for DMA")
    set_hugepages(get_hugepages() + 1)
 end
 
@@ -116,12 +117,3 @@ function selftest (options)
    print("HugeTLB page allocation OK.")
 end
 
---- ### module init: `mlock()` at load time
-
---- This module requires a stable physical-virtual mapping so this is
---- enforced automatically at load-time.
-function module_init ()
-   assert(C.lock_memory() == 0)
-end
-
-module_init()

src/core/packet.lua

@@ -100,7 +100,3 @@ function preallocate_step()
    packet_allocation_step = 2 * packet_allocation_step
 end
 
---preallocate packets freelist
-if freelist_nfree(packets_fl) == 0 then
-   preallocate_step()
-end

src/lib/checksum.c

@@ -62,7 +62,7 @@ uint16_t cksum_generic(const void *buf, size_t len, uint16_t initial)
 
 //
 // A unaligned version of the cksum,
-// n is number of 16-bit values to sum over, n in it self is a 
+// n is number of 16-bit values to sum over, n in it self is a
 // 16 bit number in order to avoid overflow in the loop
 //
 static inline uint32_t cksum_ua_loop(unsigned char *p, uint16_t n)
@@ -135,7 +135,7 @@ static inline uint32_t cksum_sse2_loop(unsigned char *p, size_t n)
 
 uint16_t cksum_sse2(unsigned char *p, size_t n, uint32_t initial)
 {
-  uint32_t sum = initial;
+  uint32_t sum = ntohs(initial);
 
   if (n < 128) { return cksum_generic(p, n, initial); }
   int unaligned = (unsigned long) p & 0xf;
@@ -183,7 +183,7 @@ static inline uint32_t cksum_avx2_loop(unsigned char *p, size_t n)
         __m256i s1 = zero;
         n -= k;
         while (k) {
-            __m256i src = _mm256_load_si256((__m256i const*) p);
+            __m256i src = _mm256_loadu_si256((__m256i const*) p);
             __m256i t;
 
             t = _mm256_unpacklo_epi8(src, zero);
@@ -217,16 +217,9 @@ static inline uint32_t cksum_avx2_loop(unsigned char *p, size_t n)
 
 uint16_t cksum_avx2(unsigned char *p, size_t n, uint32_t initial)
 {
-    uint32_t sum = initial;
+    uint32_t sum = ntohs(initial);
 
     if (n < 128) { return cksum_generic(p, n, initial); }
-    int unaligned = (unsigned long) p & 31;
-    if (unaligned) {
-        size_t k = (32 - unaligned) >> 1;
-        sum += cksum_ua_loop(p, k);
-        n -= (2*k);
-        p += (2*k);
-    }
     if (n >= 64) {
         size_t k = (n >> 5);
         sum += cksum_avx2_loop(p, k);
@@ -294,3 +287,55 @@ uint32_t tcp_pseudo_checksum(uint16_t *sip, uint16_t *dip,
   return result;
 }
 
+// calculates the initial checksum value resulting from
+// the pseudo header.
+// return values:
+// 0x0000 - 0xFFFF : initial checksum (in network order).
+// 0xFFFF0001 : unknown packet (non IPv4/6 or non TCP/UDP)
+// 0xFFFF0002 : bad header
+uint32_t pseudo_header_initial(const int8_t *buf, size_t len)
+{
+  const uint16_t const *hwbuf = (const uint16_t *)buf;
+  int8_t ipv = (buf[0] & 0xF0) >> 4;
+  int8_t proto = 0;
+  int headersize = 0;
+
+  if (ipv == 4) {           // IPv4
+    proto = buf[9];
+    headersize = (buf[0] & 0x0F) * 4;
+  } else if (ipv == 6) {    // IPv6
+    proto = buf[6];
+    headersize = 40;
+  } else {
+    return 0xFFFF0001;
+  }
+
+  if (proto == 6 || proto == 17) {     // TCP || UDP
+    uint32_t sum = 0;
+    len -= headersize;
+    if (ipv == 4) {                         // IPv4
+      if (cksum_generic_reduce(cksum_generic_loop(buf, headersize, 0)) != 0) {
+        return 0xFFFF0002;
+      }
+      sum = htons(len & 0x0000FFFF) + (proto << 8)
+              + hwbuf[6]
+              + hwbuf[7]
+              + hwbuf[8]
+              + hwbuf[9];
+
+    } else {                                // IPv6
+      sum = hwbuf[2] + (proto << 8);
+      int i;
+      for (i = 4; i < 20; i+=4) {
+        sum += hwbuf[i] +
+               hwbuf[i+1] +
+               hwbuf[i+2] +
+               hwbuf[i+3];
+      }
+    }
+    sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff);
+    sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff);
+    return sum;
+  }
+  return 0xFFFF0001;
+}

src/lib/checksum.h

@@ -23,3 +23,4 @@ void checksum_update_incremental_32(uint16_t* checksum_cell,
 uint32_t tcp_pseudo_checksum(uint16_t *sip, uint16_t *dip,
                              int addr_halfwords, int len);
 
+uint32_t pseudo_header_initial(const int8_t *buf, size_t len);

src/lib/checksum.lua

@@ -10,6 +10,7 @@ require("lib.checksum_h")
 local lib = require("core.lib")
 local ffi = require("ffi")
 local C = ffi.C
+local band = bit.band
 
 -- Select ipsum(pointer, len, initial) function based on hardware
 -- capability.
@@ -27,6 +28,22 @@ function finish_packet (buf, len, offset)
    ffi.cast('uint16_t *', buf+offset)[0] = lib.htons(ipsum(buf, len, 0))
 end
 
+function verify_packet (buf, len)
+   local initial = C.pseudo_header_initial(buf, len)
+   if     initial == 0xFFFF0001 then return nil
+   elseif initial == 0xFFFF0002 then return false
+   end
+
+   local headersize = 0
+   local ipv = band(buf[0], 0xF0)
+   if ipv == 0x60 then
+      headersize = 40
+   elseif ipv == 0x40 then
+      headersize = band(buf[0], 0x0F) * 4;
+   end
+
+   return ipsum(buf+headersize, len-headersize, initial) == 0
+end
 
 -- See checksum.h for more utility functions that can be added.
 
@@ -38,14 +55,15 @@ function selftest ()
    for i = 0, n-1 do  array[i] = i  end
    local avx2ok, sse2ok = 0, 0
    for i = 1, tests do
-      local ref =   C.cksum_generic(array+i*2, i*10+i, 0)
-      if have_avx2 and C.cksum_avx2(array+i*2, i*10+i, 0) == ref then
-	 avx2ok = avx2ok + 1
+      local initial = math.random(0, 0xFFFF)
+      local ref =   C.cksum_generic(array+i*2, i*10+i, initial)
+      if have_avx2 and C.cksum_avx2(array+i*2, i*10+i, initial) == ref then
+         avx2ok = avx2ok + 1
       end
-      if have_sse2 and C.cksum_sse2(array+i*2, i*10+i, 0) == ref then
-	 sse2ok = sse2ok + 1
+      if have_sse2 and C.cksum_sse2(array+i*2, i*10+i, initial) == ref then
+         sse2ok = sse2ok + 1
       end
-      assert(ipsum(array+i*2, i*10+i, 0) == ref, "API function check")
+      assert(ipsum(array+i*2, i*10+i, initial) == ref, "API function check")
    end
    if have_avx2 then print("avx2: "..avx2ok.."/"..tests) else print("no avx2") end
    if have_sse2 then print("sse2: "..sse2ok.."/"..tests) else print("no sse2") end

src/lib/hardware/pci.lua

@@ -74,6 +74,7 @@ end
 --- Force Linux to release the device with `pciaddress`.
 --- The corresponding network interface (e.g. `eth0`) will disappear.
 function unbind_device_from_linux (pciaddress)
+   root_check()
     local p = path(pciaddress).."/driver/unbind"
     if lib.can_write(p) then
         lib.writefile(path(pciaddress).."/driver/unbind", pciaddress)
@@ -85,6 +86,7 @@ end
 --   Pointer for memory-mapped access.
 --   File descriptor for the open sysfs resource file.
 function map_pci_memory (device, n)
+   root_check()
    local filepath = path(device).."/resource"..n
    local fd = C.open_pci_resource(filepath)
    assert(fd >= 0)
@@ -101,6 +103,7 @@ end
 --- Enable or disable PCI bus mastering. DMA only works when bus
 --- mastering is enabled.
 function set_bus_master (device, enable)
+   root_check()
    local fd = C.open_pcie_config(path(device).."/config")
    local value = ffi.new("uint16_t[1]")
    assert(C.pread(fd, value, 2, 0x4) == 2)
@@ -113,6 +116,10 @@ function set_bus_master (device, enable)
    C.close(fd)
 end
 
+function root_check ()
+   lib.root_check("error: must run as root to access PCI devices")
+end
+
 --- ### Selftest
 ---
 --- PCI selftest scans for available devices and performs our driver's

src/lib/virtio/net_device.lua

@@ -77,7 +77,7 @@ function VirtioNetDevice:new(owner)
       kickfd = {},
       virtq = {},
       rx = {},
-      tx = { 
+      tx = {
 	 p = nil,
 	 tx_mrg_hdr = ffi.new("struct virtio_net_hdr_mrg_rxbuf*[1]") ,
 	 data_sent = nil,
@@ -196,6 +196,21 @@ function VirtioNetDevice:transmit_packets_to_vm ()
    end
 end
 
+local function validflags(buf, len)
+   local valid = checksum.verify_packet(buf, len)
+
+   if valid == true then
+      return C.VIO_NET_HDR_F_DATA_VALID
+   elseif valid == false then
+      return 0
+   else
+      return C.VIO_NET_HDR_F_NEEDS_CSUM
+   end
+end
+
+
+
+
 function VirtioNetDevice:tx_packet_start(addr, len)
    local l = self.owner.input.rx
    if link.empty(l) then return nil, nil end
@@ -205,6 +220,7 @@ function VirtioNetDevice:tx_packet_start(addr, len)
 
    -- TODO: copy the relevnat fields from the packet
    ffi.fill(tx_hdr, virtio_net_hdr_size)
+   tx_hdr.flags = validflags(tx_p.data+14, tx_p.length-14)
 
    return tx_p
 end
@@ -236,20 +252,7 @@ function VirtioNetDevice:tx_packet_start_mrg_rxbuf(addr, len)
       if link.empty(l) then return end
       tx_p = link.receive(l)
 
-      -- XXX: We should validate the checksum and report the result to
-      -- the VM. -lukego
-      --
-      -- If checksum successful then set C.VIO_NET_HDR_F_DATA_VALID.
-      --
-      -- If checksum failed then set C.VIO_NET_HDR_F_NEEDS_CSUM and
-      -- let the guest do its own check to detect the error.
-      --
-      -- The call to ipsum here should be replaced with a real
-      -- IP/TCP/UDP checksum check. The current call exists only to
-      -- make the CPU do the checksumming work so that we can measure
-      -- preliminary performance.
-      checksum.ipsum(tx_p.data, tx_p.length, 0)
-      tx_mrg_hdr.hdr.flags = C.VIO_NET_HDR_F_DATA_VALID
+      tx_mrg_hdr.hdr.flags = validflags(tx_p.data+14, tx_p.length-14)
 
       self.tx.tx_mrg_hdr[0] = tx_mrg_hdr
       self.tx.data_sent = 0

src/program/snabbnfv/README

@@ -6,3 +6,5 @@ Usage:
   snabbnfv fuzz
 
 Use --help for per-command usage.
+Example: 
+    snabbnfv traffic --help