diff --git a/.gitignore b/.gitignore index 1785f8349f..cf01804622 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ obj /src/syscall.lua /src/syscall /deps/*.vsn +/src/program/programs.inc diff --git a/src/Makefile b/src/Makefile index 8b2c4bafba..33e090988b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -191,9 +191,9 @@ $(INCOBJ): obj/%_inc.o: %.inc Makefile program/programs.inc | $(OBJDIR) # Create list of programs that exist program/programs.inc: - @ls -1d program/*/ | xargs basename -a > $@ + @(for d in program/*/; do basename $$d; done) > $@ -.FORCE: program/programs.inc +FORCE: # extra/ third party bits and pieces obj/strict.o: extra/strict.lua | $(OBJDIR) diff --git a/src/arch/avx2.c b/src/arch/avx2.c index a3a6cde8cd..b43acf496c 100644 --- a/src/arch/avx2.c +++ b/src/arch/avx2.c @@ -63,7 +63,7 @@ static inline uint32_t cksum_avx2_loop(unsigned char *p, size_t n) uint16_t cksum_avx2(unsigned char *p, size_t n, uint32_t initial) { - uint32_t sum = ntohs(initial); + uint32_t sum = initial; if (n < 128) { return cksum_generic(p, n, initial); } if (n >= 64) { diff --git a/src/arch/sse2.c b/src/arch/sse2.c index 9551c80aa4..b8d86b35c0 100644 --- a/src/arch/sse2.c +++ b/src/arch/sse2.c @@ -66,7 +66,7 @@ static inline uint32_t cksum_sse2_loop(unsigned char *p, size_t n) uint16_t cksum_sse2(unsigned char *p, size_t n, uint32_t initial) { - uint32_t sum = ntohs(initial); + uint32_t sum = initial; if (n < 128) { return cksum_generic(p, n, initial); } int unaligned = (unsigned long) p & 0xf; diff --git a/src/core/main.lua b/src/core/main.lua index 774a5cd435..91dad36285 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -59,7 +59,7 @@ end function usage () print("Usage: "..ffi.string(C.argv[0]).." ...") - local programs = require("program.programs_inc"):gsub("[a-z]+", " %1") + local programs = require("program.programs_inc"):gsub("%S+", " %1") print() print("This snabb executable has the following programs built in:") print(programs) diff --git a/src/core/snabbswitch.c b/src/core/snabbswitch.c index 2b1ac5ddbc..4e0abd93ba 100644 --- a/src/core/snabbswitch.c +++ b/src/core/snabbswitch.c @@ -14,6 +14,6 @@ int main(int snabb_argc, char **snabb_argv) argv = snabb_argv; lua_State* L = luaL_newstate(); luaL_openlibs(L); - return luaL_dostring(L, "require \"core.main\""); + return luaL_dostring(L, "require \"core.startup\""); } diff --git a/src/core/startup.lua b/src/core/startup.lua new file mode 100644 index 0000000000..6ce0c8186f --- /dev/null +++ b/src/core/startup.lua @@ -0,0 +1,6 @@ +local ok, err = pcall(require, "core.main") +if not ok then + print("startup: unhandled exception") + print(err) + os.exit(1) +end diff --git a/src/doc/branches.md b/src/doc/branches.md index 5535e17e64..588d039642 100644 --- a/src/doc/branches.md +++ b/src/doc/branches.md @@ -11,14 +11,36 @@ The current state of each branch with respect to master is visible here: #### master BRANCH: master git://github.com/lukego/snabbswitch - Synchronization point with changes recommended for all users. + Stable branch suitable for development and deployment. - - Makes monthly releases. - - Changes are gated by the SnabbBot CI - - Currently Linux/x86-64 with GNU tools (more in the future) + - Always contains a stable release that is safe to pull from. + - Updated monthly with new features and weekly with new bug fixes. + - Changes are gated by the SnabbBot CI. Maintainer: Luke Gorrie and Max Rottenkolber +#### next + + BRANCH: next git://github.com/lukego/snabbswitch + Test and integration branch for new development. + + - Contains the changes for the next monthly feature release. + - Merges Pull Requests that pass code review on Github. + - Cycles between unstable and stable with the release schedule. + + Maintainer: Luke Gorrie + +#### fixes + + BRANCH: fixes git://github.com/lukego/snabbswitch + Test and integration branch for bug fixes. + + - Contains the changes for the next weekly maintenance release. + - Merges Pull Requests that fix bugs in the latest release. + - Generally stable. + + Maintainer: Luke Gorrie + #### documenation-fixups BRANCH: documentation-fixups git://github.com/eugeneia/snabbswitch diff --git a/src/doc/documentation-guide.mk2 b/src/doc/documentation-guide.mk2 new file mode 100644 index 0000000000..0eb901bbd9 --- /dev/null +++ b/src/doc/documentation-guide.mk2 @@ -0,0 +1,202 @@ +# Contributing Documentation in SnabbSwitch + +## README.md, README.md.src and README(.inc) + +The SnabbSwitch documentation is organized in README's spread across the +code base. If you commit changes you should look for the "nearest" README +file related to the modules you changed and update these. There are three +different kinds of README files: + +* `README.md` — A portion of the SnabbSwitch manual, embedded by GitHub + too. These are often (but not always) artifacts built from a `.src` + file. Edit these if no `.src` is available (see below). +* `README.md.src` — A build recipe. If available, this is the one you + must edit. These are formatted in [GitHub Flavored + Markdown](https://help.github.com/articles/github-flavored-markdown/). +* `README(.inc)` — Plain text files included as the `--help` message of + SnabbSwitch programs. These should be using only the ASCII character + set to ensure compatibility with older terminals. + +For instance if you had changed the API of `lib.protocol.ethernet` you'd +need to update the documentation in `lib/protocol/README.md.src`. It is +important that you use the correct header level (e.g. `##`, `###`, +...). Which level you are at can be seen in `doc/genbook.sh` (see +*Building A Standalone Documentation*). + +Be sure to know who your target audience is. We recognize three different +groups: + +* **Casual users:** Some programs are targeting *anyone* using Snabb + Switch, make sure that their `--help` message (`README.inc`) does not + require prior knowledge. +* **Power users:** Public APIs like `core.packet` or + `lib.protocol.ethernet` are there for app designers to use. These + should be well documented but may require knowledge of Lua and other + parts of the Snabb Switch system. +* **Contributors:** Some documents (like the one you are reading now) are + facing towards Snabb Switch contributors. They can be frequently + updating and even contain open questions. We should ensure that these + always reflect the current state of affairs. + +### Building README.md Files + +In order to recreate a `README.md` file from its `.src` you need to +`make` it. E.g. in case of our example above you would run: + +``` +make lib/protocol/README.md +``` + +You need to commit the resulting `README.md` (and possibly generated +diagram images, see *Including Diagrams*) alongside the updated +`README.md.src`. + +### Including Diagrams + +The main reason for the `README.md.src` to `README.md` step is that we +use a custom Markdown pre-processor to be able to embed ASCII diagrams in +our documentation. These ASCII diagrams will be rendered to pretty images +by [ditaa](http://ditaa.sourceforge.net/). In order to build `README.md` +files containing diagrams you will need a local installation of ditaa. + +In order to use the diagram pre-processor you need to embed a ditaa +diagram in a block *indented with four space characters* lead by a +`DIAGRAM: ` header: + +``` +Normal paragraph... + + DIAGRAM: My Diagram + +------+ + |A Box |<--With an arrow + +------+ +``` + +## Building A Standalone Documentation + +In order to build a complete SnabbSwitch reference manual you can use +`make doc/snabbswitch.html`, `make doc/snabbswitch.epub` and `make +doc/snabbswitch.pdf` to build the HTML, ePUB and PDF versions +respectively. Alternatively, `make book` can be used to build all three +at once. For these tasks to work you will need a local installation of +[Pandoc](http://johnmacfarlane.net/pandoc/). + +On Ubuntu you can install everything required to produce HTML, PDF and +epub versions with the following `apt-get` command: + +``` +sudo apt-get install pandoc pandoc-citeproc texlive-latex-recommended texlive-xetex texlive-luatex texlive-fonts-recommended texlive-latex-extra texlive-fonts-extra +``` + +You can change the organization and contents of the resulting file by +editing `doc/genbook.sh`, which is really just a primitive shell script +that concatenates the various `README.md` files we have. + +# Stylistic Conventions + +## Anatomy Of A Module Section + +Every module has its own subsection in the SnabbSwitch manual, and all +these sections start off the same way: + +``` +### Protocol Header (lib.protocol.header) + +The `lib.protocol.header` module contains stuff... +``` + +That is: The header contains the title of the module as well as its +*module path* in parentheses. The header is followed by a paragraph that +again names the module path and summarizes the module's purpose. This +introduction can be as detailed as the module required. Some modules are +obvious, some deserve along-form high-level introduction with examples. + +If the module in question is an App, the introduction must be followed by +a diagram visually describing its inputs and outputs. E.g.: + +``` + DIAGRAM: MyApp + +------------+ + | | + in ---->* MyApp *----> out + | | + +------------+ +``` + +After the introduction follows a complete description of every *external* +symbol of the module. External means symbols that are part of the modules +public API. Every symbol gets its own special mention of the form: + +``` +— <Type> **<qualified.name>** <type-specific> + +Paragraphs describing the symbol... +``` + +The `—` character is an *em dash*. Currently we use the following types: +Variable, Function, Method and Key. Variable and function names are +prefixed with their module name (separated from the symbol name by a +`.`). Methods are prefixed with their class name (separated from the +symbol name by a `:`). Functions and methods are followed by their +parameter lists. E.g.: + +``` +— Variable **module.some_constant** + +— Function **module.some_function** *arg1*, *arg2* + +— Method **class:some_method** *arg1*, *arg2* +``` + +If the module in question is an App, the symbol definitions must be +followed by a sub-section "Configuration" that elaborates on the App's +configuration parameter. E.g. + +``` +### Configuration + +The `nd_light` app accepts a table as its configuration argument. The +following keys are defined: + +— Key **local_mac** + +*Required*. Local MAC address as a string or in binary representation. + +— Key **retrans** + +*Optional*. Number of neighbor solicitation retransmissions. Default is +unlimited retransmissions. +``` + +Each key's description must be preceded by either `*Required*` or +`*Optional*` to signify if its a required or an optional parameter. Each +key's description must also declare the expected type of the argument +value. Each optional key's description must end in a sentence defining +its default value. + + +## Markup Conventions + +We markup source code literals in `code` font. E.g.: "The `foobar` module +is nice" and "`mod.fun(bla)` will make your dreams come true". Parameter +identifiers are marked up in *italic* font. E.g.: "`mod.foo` takes an +argument *bar*". + +UNIX system calls should be mentioned like so: `usleep(3)`. + +We markup specific *concepts* we introduce in italic font the first time +they are mentioned in order to signify to the reader that a specific +concept has a well defined meaning. + +## Terminology And Normalized Language + +The parameter names used in method and function description do not need +to reflect the names used in the source code. Instead use long, +descriptive names made out of full words when sensible. + +Symbol definition are written in third person, e.g.: "Returns a number" +instead of "Return a number". When describing default behavior we say +"The default is..." instead of "Defaults to..." etc. + +When in doubt, turn to the [Lua Reference Manual](http://www.lua.org/manual/5.1/) +for linguistic and stylistic reference. \ No newline at end of file diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md new file mode 100644 index 0000000000..353384cf3e --- /dev/null +++ b/src/doc/git-workflow.md @@ -0,0 +1,86 @@ +# Snabb Switch Git Workflow + +This document explains the Git workflows that we use to develop and +release Snabb Switch. + +## Overview + +Snabb Switch development follows a few well-known patterns: + +- We follow the distributed development model [described by Linus + Torvald](https://www.youtube.com/watch?v=4XpnKHJAok8) and pioneered + by the Linux kernel. +- We use a [merge based workflow](https://www.atlassian.com/git/articles/git-team-workflows-merge-or-rebase/). +- We use the [fork and pull](https://help.github.com/articles/using-pull-requests/#fork--pull) + model of collaboration. +- We aim to support ad-hoc collaboration inspired by the + [DMZ Flow](https://gist.github.com/djspiewak/9f2f91085607a4859a66). + +## HOWTO + +### Download and update the latest release + +1. Clone the [SnabbCo/snabbswitch](/~https://github.com/SnabbCo/snabbswitch) repository. +2. Check out and build the `master` branch. +3. Pull when you want to update to the latest stable release. + +### Develop and contribute an improvement + +1. [Create your own fork](https://help.github.com/articles/fork-a-repo/) of Snabb Switch on Github. +2. Develop and debug your contribution on a new [topic branch](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows#Topic-Branches) based on the latest `master`. +3. Make a final cleanup of your code before review. (Last chance to rebase.) +4. Submit a Github [Pull Request](https://help.github.com/articles/using-pull-requests/#initiating-the-pull-request) + to the `master` branch. +5. Respond to feedback and correct problems by pushing additional commits. + +There are two milestones in the process of accepting your change: + +1. Your change is merged onto a branch that feeds `master`, for + example `next`, `fixes`, `documentation-fixes`, or `nfv`. From this + point the owner of that branch will push your work upstream + together with other related changes. They might ask you for help + but otherwise your work is done. +2. Your change is merged onto `master`. This could happen in a series + of merge steps, for example `nfv->next->master`. Once this happens + your code has been officially released as part of Snabb Switch. + +### Develop and maintain a new program + +Snabb Switch includes programs like `snabbnfv`, `packetblaster`, and +`snsh`. Here is how you can create a new program and take charge of +its development. + +1. [Fork](https://help.github.com/articles/fork-a-repo/) your own + repository on Github. +2. Create a [long-lived branch](branches.md) where new development of your program will be done. +3. Create a directory `src/program/myapplication/` and develop your program. +4. `git merge master` regularly to stay synchronized with the main line of development. +5. Optional: Send releases of your application to `master` with Pull Requests. + +The code in your `src/program/myapplication/` directory is developed +according to your own rules and tastes. If there are parts of this +code that you especially want to have reviewed (or do not want to have +reviewed) then please explain this in your Pull Request. The only +necessary review is to make sure that programs do not negatively +impact each other or change shared code without enough review. + +Pull Requests that make changes to your application will be referred +to you for merge onto your branch. + +Use the *Develop and contribute an improvement* workflow to make +changes to the core Snabb Switch code. Please do not bundle +substantial changes to the core software with updates to your program. + +If you do not want to include your program in the main Snabb Switch +release then this is no problem. You can simply pull from `master` to +receive updates and skip the step of pushing back. + +### To help maintain Snabb Switch + +Here are the best ways to help maintain Snabb Switch: + +1. Review Pull Requests to help people quickly improve them. +2. Test the `next` branch and help fix problems before releases. +3. Contribute new `selftest` cases to make our CI more effective. +4. Maintain a [branch](branches.md) where you accept Pull Requests and push them upstream. + diff --git a/src/lib/checksum.c b/src/lib/checksum.c index d70af86889..73c98043b2 100644 --- a/src/lib/checksum.c +++ b/src/lib/checksum.c @@ -14,11 +14,10 @@ #include <stdint.h> #include <sys/time.h> -static inline uint32_t cksum_generic_loop(const void *buf, size_t len, uint32_t sum) +uint16_t cksum_generic(unsigned char *p, size_t len, uint16_t initial) { - /* workaround gcc strict-aliasing warning */ - uintptr_t ptr = (uintptr_t)buf; - const uint16_t *u16 = (const uint16_t *)ptr; + uint32_t sum = htons(initial); + const uint16_t *u16 = (const uint16_t *)p; while (len >= (sizeof(*u16) * 4)) { sum += u16[0]; @@ -38,22 +37,9 @@ static inline uint32_t cksum_generic_loop(const void *buf, size_t len, uint32_t if (len == 1) sum += *((const uint8_t *)u16); - return sum; -} - -static inline uint16_t cksum_generic_reduce(uint32_t sum) -{ - sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff); - sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff); - return (uint16_t)~sum; -} - -uint16_t cksum_generic(const void *buf, size_t len, uint16_t initial) -{ - uint32_t sum; - - sum = cksum_generic_loop(buf, len, initial); - return htons(cksum_generic_reduce(sum)); + while(sum>>16) + sum = (sum & 0xFFFF) + (sum>>16); + return ntohs((uint16_t)~sum); } // SIMD versions @@ -152,7 +138,7 @@ uint32_t pseudo_header_initial(const int8_t *buf, size_t len) uint32_t sum = 0; len -= headersize; if (ipv == 4) { // IPv4 - if (cksum_generic_reduce(cksum_generic_loop(buf, headersize, 0)) != 0) { + if (cksum_generic((unsigned char *)buf, headersize, 0) != 0) { return 0xFFFF0002; } sum = htons(len & 0x0000FFFF) + (proto << 8) diff --git a/src/lib/checksum.h b/src/lib/checksum.h index 677addddeb..551cb31792 100644 --- a/src/lib/checksum.h +++ b/src/lib/checksum.h @@ -1,14 +1,14 @@ // Calculate IP checksum using SSE2 instructions. // (This will crash if you call it on a CPU that does not support SSE.) -uint16_t cksum_sse2(unsigned char *p, size_t n, uint32_t initial); +uint16_t cksum_sse2(unsigned char *p, size_t n, uint16_t initial); // Calculate IP checksum using AVX2 instructions. // (This will crash if you call it on a CPU that does not support AVX2.) -uint16_t cksum_avx2(unsigned char *p, size_t n, uint32_t initial); +uint16_t cksum_avx2(unsigned char *p, size_t n, uint16_t initial); // Calculate IP checksum using portable C code. // This works on all hardware. -uint16_t cksum_generic(unsigned char *p, size_t n, uint32_t initial); +uint16_t cksum_generic(unsigned char *p, size_t n, uint16_t initial); // Incrementally update checksum when modifying a 16-bit value. void checksum_update_incremental_16(uint16_t* checksum_cell, diff --git a/src/lib/checksum.lua b/src/lib/checksum.lua index 782373ed43..d590e1f837 100644 --- a/src/lib/checksum.lua +++ b/src/lib/checksum.lua @@ -2,7 +2,30 @@ module(...,package.seeall) -- This module exposes the interface: -- checksum.ipsum(pointer, length, initial) => checksum --- where checksum is in network byte order. +-- +-- pointer is a pointer to an array of data to be checksummed. initial +-- is an unsigned 16-bit number in host byte order which is used as +-- the starting value of the accumulator. The result is the IP +-- checksum over the data in host byte order. +-- +-- The initial argument can be used to verify a checksum or to +-- calculate the checksum in an incremental manner over chunks of +-- memory. The synopsis to check whether the checksum over a block of +-- data is equal to a given value is the following +-- +-- if ipsum(pointer, length, value) == 0 then +-- -- checksum correct +-- else +-- -- checksum incorrect +-- end +-- +-- To chain the calculation of checksums over multiple blocks of data +-- together to obtain the overall checksum, one needs to pass the +-- one's complement of the checksum of one block as initial value to +-- the call of ipsum() for the following block, e.g. +-- +-- local sum1 = ipsum(data1, length1, 0) +-- local total_sum = ipsum(data2, length2, bit.bnot(sum1)) -- -- The actual implementation is chosen based on running CPU. diff --git a/src/lib/protocol/gre.lua b/src/lib/protocol/gre.lua index d11d1c8e53..25f7621598 100644 --- a/src/lib/protocol/gre.lua +++ b/src/lib/protocol/gre.lua @@ -3,7 +3,8 @@ local ffi = require("ffi") local C = ffi.C local header = require("lib.protocol.header") local lib = require("core.lib") -local bitfield, update_csum, finish_csum = lib.bitfield, lib.update_csum, lib.finish_csum +local bitfield = lib.bitfield +local ipsum = require("lib.checksum").ipsum -- GRE uses a variable-length header as specified by RFCs 2784 and -- 2890. The actual size is determined by flag bits in the base @@ -106,8 +107,9 @@ local function checksum(header, payload, length) local csum_in = header.csum; header.csum = 0; header.reserved1 = 0; - local csum = finish_csum(update_csum(payload, length, - update_csum(header, ffi.sizeof(header), 0))) + local csum = ipsum(payload, length, + bit.bnot(ipsum(ffi.cast("uint8_t *", header), + ffi.sizeof(header), 0))) header.csum = csum_in return csum end diff --git a/src/lib/protocol/icmp/header.lua b/src/lib/protocol/icmp/header.lua index 2218809ea7..635b67ed31 100644 --- a/src/lib/protocol/icmp/header.lua +++ b/src/lib/protocol/icmp/header.lua @@ -3,6 +3,7 @@ local ffi = require("ffi") local C = ffi.C local header = require("lib.protocol.header") local lib = require("core.lib") +local ipsum = require("lib.checksum").ipsum -- XXX IPv4 and IPv6 use the same ICMP header format but distinct -- number spaces for type and code. This class needs to be subclassed @@ -59,16 +60,16 @@ local function checksum(header, payload, length, ipv6) if ipv6 then -- Checksum IPv6 pseudo-header local ph = ipv6:pseudo_header(length + ffi.sizeof(header), 58) - csum = lib.update_csum(ph, ffi.sizeof(ph), csum) + csum = ipsum(ffi.cast("uint8_t *", ph), ffi.sizeof(ph), 0) end -- Add ICMP header local csum_rcv = header.checksum header.checksum = 0 - csum = lib.update_csum(header, ffi.sizeof(header), csum) + csum = ipsum(ffi.cast("uint8_t *", header), + ffi.sizeof(header), bit.bnot(csum)) header.checksum = csum_rcv -- Add ICMP payload - csum = lib.update_csum(payload, length, csum) - return lib.finish_csum(csum) + return ipsum(payload, length, bit.bnot(csum)) end function icmp:checksum (payload, length, ipv6) diff --git a/src/lib/protocol/ipv4.lua b/src/lib/protocol/ipv4.lua index 338fe3feb6..1bff53d1f9 100644 --- a/src/lib/protocol/ipv4.lua +++ b/src/lib/protocol/ipv4.lua @@ -3,6 +3,7 @@ local ffi = require("ffi") local C = ffi.C local lib = require("core.lib") local header = require("lib.protocol.header") +local ipsum = require("lib.checksum").ipsum -- TODO: generalize local AF_INET = 2 @@ -143,8 +144,8 @@ function ipv4:protocol (protocol) end function ipv4:checksum () - local csum = lib.update_csum(self:header(), self:sizeof()) - self:header().checksum = C.htons(lib.finish_csum(csum)) + self:header().checksum = C.htons(ipsum(ffi.cast("uint8_t *", self:header()), + self:sizeof(), 0)) return C.ntohs(self:header().checksum) end diff --git a/src/lib/protocol/tcp.lua b/src/lib/protocol/tcp.lua index debe3d301b..cb1212395f 100644 --- a/src/lib/protocol/tcp.lua +++ b/src/lib/protocol/tcp.lua @@ -3,6 +3,7 @@ local ffi = require("ffi") local C = ffi.C local lib = require("core.lib") local header = require("lib.protocol.header") +local ipsum = require("lib.checksum").ipsum local tcp_header_t = ffi.typeof[[ struct { @@ -146,14 +147,14 @@ function tcp:checksum (payload, length, ip) if ip then -- Checksum IP pseudo-header local ph = ip:pseudo_header(length + self:sizeof(), 6) - csum = lib.update_csum(ph, ffi.sizeof(ph), csum) + csum = ipsum(ffi.cast("uint8_t *", ph), ffi.sizeof(ph), 0) end -- Add TCP header h.checksum = 0 - csum = lib.update_csum(h, self:sizeof(), csum) + csum = ipsum(ffi.cast("uint8_t *", h), + self:sizeof(), bit.bnot(csum)) -- Add TCP payload - csum = lib.update_csum(payload, length, csum) - h.checksum = C.htons(lib.finish_csum(csum)) + h.checksum = C.htons(ipsum(payload, length, bit.bnot(csum))) end return C.ntohs(h.checksum) end diff --git a/src/lib/protocol/udp.lua b/src/lib/protocol/udp.lua index 445b1a055a..7566f85b2e 100644 --- a/src/lib/protocol/udp.lua +++ b/src/lib/protocol/udp.lua @@ -2,6 +2,7 @@ module(..., package.seeall) local ffi = require("ffi") local C = ffi.C local header = require("lib.protocol.header") +local ipsum = require("lib.checksum").ipsum local udp_header_t = ffi.typeof[[ struct { @@ -64,14 +65,14 @@ function udp:checksum (payload, length, ip) if ip then -- Checksum IP pseudo-header local ph = ip:pseudo_header(length + self:sizeof(), 17) - csum = lib.update_csum(ph, ffi.sizeof(ph), csum) + csum = ipsum(ffi.cast("uint8_t *", ph), ffi.sizeof(ph), 0) end -- Add UDP header h.checksum = 0 - csum = lib.update_csum(h, self:sizeof(), csum) + csum = ipsum(ffi.cast("uint8_t *", h), + self:sizeof(), bit.bnot(csum)) -- Add UDP payload - csum = lib.update_csum(payload, length, csum) - h.checksum = C.htons(lib.finish_csum(csum)) + h.checksum = C.htons(ipsum(payload, length, bit.bnot(csum))) end return C.ntohs(h.checksum) end diff --git a/src/program/snabbnfv/doc/.images/compute-node.png b/src/program/snabbnfv/doc/.images/compute-node.png new file mode 100644 index 0000000000..8402dc7dcb Binary files /dev/null and b/src/program/snabbnfv/doc/.images/compute-node.png differ diff --git a/src/program/snabbnfv/doc/.images/database-node.png b/src/program/snabbnfv/doc/.images/database-node.png new file mode 100644 index 0000000000..62084cf15b Binary files /dev/null and b/src/program/snabbnfv/doc/.images/database-node.png differ diff --git a/src/program/snabbnfv/doc/.images/network-node.png b/src/program/snabbnfv/doc/.images/network-node.png new file mode 100644 index 0000000000..8b2405832d Binary files /dev/null and b/src/program/snabbnfv/doc/.images/network-node.png differ diff --git a/src/program/snabbnfv/doc/.images/neutron-map.png b/src/program/snabbnfv/doc/.images/neutron-map.png new file mode 100644 index 0000000000..30f4fde948 Binary files /dev/null and b/src/program/snabbnfv/doc/.images/neutron-map.png differ diff --git a/src/program/snabbnfv/doc/architecture.md b/src/program/snabbnfv/doc/architecture.md new file mode 100644 index 0000000000..83a715f535 --- /dev/null +++ b/src/program/snabbnfv/doc/architecture.md @@ -0,0 +1,70 @@ +# Snabb NFV Architecture + +[Snabb NFV](http://snabb.co/nfv.html) is deployed for OpenStack with +components on the Network Node, the Database Node, and the Compute +Nodes. + +![Snabb NFV: overview](.images/neutron-map.png) + +The design goal is to distribute a consistent snapshot of the Neutron +network configuration to every node at a reasonable interval (e.g. once +per second) and to behave in a reasonable and predictable way when any +node fails, restarts, or slows down. + +## Network node + +![Snabb NFV: network node](.images/network-node.png) + +The **Snabb NFV ML2 Mechanism Driver** is deployed on the Network +Node. This is a Python module that runs inside OpenStack Neutron. The +Snabb mechanism driver implements *port binding* to assign selected ports +to Snabb NFV. You can also run other mechanism drivers in parallel and +bind other ports to those. + +The Snabb NFV mechanism driver is most similar to the OpenDaylight +mechanism driver. + +## Database node + +![Snabb NFV: database node](.images/database-node.png) + +The **Snabb NFV Sync Master** runs on the database node. This daemon +periodically captures a consistent snapshot of the Neutron configuration +and makes it available for synchronization over the network. + +The sync master is implemented by a shell script called +[neutron-sync-master](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/neutron_sync_master). +The database is periodically snapshotted into CSV files with `mysqldump` +and published for synchronization using `git`. The snapshot interval is +configurable and defaults to once per second. + +## Compute node + +![Snabb NFV: compute node](.images/compute-node.png) + +Each compute node runs the Sync Agent and one or more Traffic processes. + +The **Snabb NFV Sync Agent** periodically polls the Sync Master for an +updated Neutron configuration. Each time the Neutron configuration +changes the Sync Agent generates new configuration files for each of the +Snabb NFV Traffic processes that are affected. + +The sync agent is a shell script called +[neutron-sync-agent](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/neutron_sync_agent) +and uses the +[neutron2snabb](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/neutron2snabb) +program to translate the master Neutron configuration into individual +configuration files for each local traffic process. + +Each **Snabb NFV Traffic** process performs packet processing between one +physical network port (PCI device) and all of the virtual machines +connected to that port. Each traffic process loads its configuration from +an individual file in its own native format. When the configuration file +changes the traffic process automatically loads the new version. Each +traffic process is identified by the PCI address of the network device +that it operates, for example `07:00.0`. + +For the standard high-performance deployment scenario each traffic +process runs on a dedicated CPU core. For peak performance the traffic +process should be assigned a CPU core and PCI network device that both +belong to the same NUMA node. diff --git a/src/program/snabbnfv/doc/compute-node-requirements.md b/src/program/snabbnfv/doc/compute-node-requirements.md new file mode 100644 index 0000000000..535b15ee94 --- /dev/null +++ b/src/program/snabbnfv/doc/compute-node-requirements.md @@ -0,0 +1,48 @@ +# Compute Node Requirements + +The Snabb NFV data plane has specific hardware and kernel setup +requirements for compute nodes. These requirements are based on the +standard recommendations from Intel for high performance data planes (see +[Intel DPDK](http://dpdk.org/doc/intel/dpdk-start-linux-1.7.0.pdf) +recommendations for more background.) + +#### Hardware setup: + +1. Network interfaces based on Intel 82599 Ethernet controller. +2. Network cards in suitable PCIe slots (typically PCIe 2.0/3.0 x8). +3. Network cards connected to CPUs (NUMA nodes) as desired. For example, +spread equally between nodes. + +#### Reserve CPU cores for Snabb Switch + +Snabb Switch traffic processes run on dedicated CPU cores. The peak +performance configuration is to reserve one core (and its hyperthread) +for each 10G network port. + +For example, on the server: + +* 2 x Xeon E5-2699v3 CPUs (36 cores / 72 threads) +* 8 x 10G ports (4 per CPU / NUMA node) + +You could reserve these resources: + +* CPU cores 0-3 (CPU0) and 18-21 (CPU1). +* Corresponding Hyperthreads 36-39 (CPU0) and 54-57 (CPU1). (If + Hyperthreads are enabled in hardware.) + +The details of how to reserve CPUs and assign them to Snabb Switch +processes are given below. + +#### Kernel parameters setup + +1. Disable the IOMMU: `intel_iommu=off`. +2. Reserve all memory for virtual machines as huge pages. Example for +24GB x 2MB pages: `hugepages=12288`. +3. Reserve CPU cores for Snabb Switch traffic processes by isolating them +from the Linux scheduler. Based on the example above: +`isolcpus=0-3,18-21,36-39,54-57`. + +On Ubuntu 14.04 this could be accomplished with this line in +`/etc/default/grub`: + + GRUB_CMDLINE_LINUX_DEFAULT="hugepages=12288 intel_iommu=off isolcpus=0-3,18-21,36-39,54-57" diff --git a/src/program/snabbnfv/doc/installation.md b/src/program/snabbnfv/doc/installation.md new file mode 100644 index 0000000000..b72e42d5b8 --- /dev/null +++ b/src/program/snabbnfv/doc/installation.md @@ -0,0 +1,87 @@ +# Installation + +Here are some tips for how to relate the software +[Architecture](architecture.md) to nuts and bolts like files to install +and init scripts to write. + +## Software installation + +There is only one file to install: the `snabb` executable. + +To create this file you can checkout the +[snabbswitch](/~https://github.com/SnabbCo/snabbswitch/) repository and run +`make` at the top level to produce `src/snabb`. You can install this in a +standard location such as `/usr/bin/snabb`. + +`snabb` is a stand-alone executable that contains all of the required +functions as sub-commands (in the style of busybox). + +The relevant usages are: + +* [`snabb snabbnfv traffic ...`](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/traffic) + on the compute node does the traffic processing. + +* [`snabb snabbnfv neutron-sync-master ...`](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/neutron_sync_master) + on the database node(s) makes the Neutron configuration available to + compute nodes. + +* [`snabb snabbnfv neutron-sync-agent ...`](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/neutron_sync_agent) + on the compute nodes to poll the master for configuration updates. + +* [`snabb snabbnfv neutron2snabb ...`](/~https://github.com/SnabbCo/snabbswitch/tree/master/src/program/snabbnfv/neutron2snabb) + to translate the Neutron database into configuration files for the local traffic processes. This command is called automatically from `neutron-sync-agent`. + +The `traffic` command has no external software dependencies. Other +commands expect certain programs to be in the `PATH` to call: `snabb`, +`git`, `mysqldump`, `diff`, and standard shellutils. + +## What to run + +These are the services that should run in addition to OpenStack itself: + +* `neutron-sync-master`: One instance per database server. +* `neutron-sync-agent`: One instance per compute node. +* `traffic`: One instance per 10G traffic port on each compute node. + +The command line arguments for each process can be determined by reading +the usage and considering whether the default is suitable. + +The processes print log messages to stdout and these should be redirected +to a suitable location such as syslog. + +## Traffic process + +### Configuration + +You do not have to write configuration files for the traffic +application. These are generated automatically by `neutron_sync_agent` +based on the Neutron database configuration. + +### CPU affinity + +The traffic process should run on a reserved CPU core (see [[Compute node +requirements]]). The core can be assigned with `taskset -c <core> snabb +snabbnfv traffic ...`. + +Note: The command `numactl` seems to be incompatible with the Linux +`isolcpus` feature and for this reason it is not recommended. + +The designated core should be selected from a NUMA node that corresponds +to the PCI device the traffic process is attaching to. Checking the NUMA +node assignment for device 0000:00:03. can be done like this: + +``` +cat /sys/bus/pci/devices/0000:00:03.0/numa_node +``` + +### `traffic` restarts + +The traffic process is designed to be safe to restart. If it detects an +error it will terminate with a message and expect to be restarted. Once +restarted it will continue serving traffic for the virtual +machines. (Stateful packet filtering connection tables are reset during +restarts, however.) + +We recommend that the traffic process is always restarted automatically +when it terminates, after some reasonable delay (e.g. between one and ten +seconds). diff --git a/src/program/snabbnfv/doc/neutron-api-extensions.md b/src/program/snabbnfv/doc/neutron-api-extensions.md new file mode 100644 index 0000000000..45bed68839 --- /dev/null +++ b/src/program/snabbnfv/doc/neutron-api-extensions.md @@ -0,0 +1,143 @@ +# Neutron API Extentions + +## Zones + +Snabb NFV introduces a new Neutron network type: `zone`. + +Zones are like VLANs but more abstract. Whereas a `type:vlan` network +corresponds directly with a Layer-2 network segment, a `type:zone` +network can be realized in an operator-specific way. Zones thus provide a +framework for operators to connect virtual machines to networks that are +not traditionally supported by Neutron. + +Zone networks are defined using the familiar Neutron API: + +``` +neutron net-create --provider:network_type=zone \ + --provider:segmentation-id=<zone-number> +``` + +The actual implementation of Zones for a specific operator's network is +determined by an ML2 mechanism driver. Snabb NFV includes one such +driver: `piesss`. + +### Zones mapped to TeraStream + +Snabb NFV supports mapping zones onto Deutsche Telekom's +[TeraStream](https://ripe67.ripe.net/archives/video/3/) network. + +TeraStream has these key characteristics: + +* Simple unified Layer-3 (IPv6) network. +* 64 different services: infrastructure, voice, consumer internet, etc. +* Service type encoded into IPv6 address (`PIESSS` coding). +* Each server port has 64 unique IPv6 subnets (one for each service). + +The main challenge for Neutron is that TeraStream uses separate IPv6 +subnets for each physical port. This means that no suitable IP address +can be assigned to a Neutron port at the time it is created, which would +be the standard Neutron behavior. Instead the IP address must be assigned +after the Neutron Port is bound to a physical server port. + +Snabb NFV addresses this issue and connects Neutron ports to TeraStream +as follows: + +* Define one Network for each of the 64 zones. +* Define a template subnet for each network, for example `0::0/64`. +* Ports are allocated template addresses at creation time, for example + `0::42/64`. The template address is stored in the Port `fixed_ips` + attribute. +* Ports are assigned real addresses at portbinding time. The real address + is determined by combining the Port's template IP address (for low + address bits) with the appropriate subnet for the chosen physical port + (for high address bits). The result is the real address and this is + stored in the `zone_ip` field of the Port `vif_details` attribute. +* Ports are also mapped to specific VLAN-IDs to match their real subnets. + +## Bandwidth reservation + +Snabb NFV associates a bandwidth reservation (Gbps) with each Port when +it is created. + +The bandwidth reservation is taken into account when choosing which +physical port of a server to use for a Neutron port. This is to ensure +that the physical network capacity is appropriately shared with the +virtual machines based on their expected bandwidth needs. (If a server +has 8 x 10Gbps physical ports then it would be unfortunate to choose the +same physical port for two Neutron Ports that each require 10Gbps of +bandwidth.) + +The operator specifies a bandwidth reservation at Port creation time +using the `binding:profile` attribute. Here is an example of a 6 Gbps +reservation: + +``` +neutron port-create ... --binding:profile type=dict zone_gbps=6 +``` + +(The default reservation is 1 Gbps.) + +### Port selection based on bandwidth reservation + +The physical port will be chosen in one of two ways, depending on whether +bandwidth is oversubscribed: + +1. If one or more physical ports has enough bandwidth available to +support the new reservation without becoming oversubscribed, then the +most heavily loaded of these ports is chosen. That is, the first priority +is to fill up the ports that are already partly used. + +2. If the requested bandwidth is not readily available, and the server +will be oversubscribed with the new reservation, then the least loaded +port is chosen. That is, the second priority is to share the load as +evenly as possible on servers that are oversubscribed. + +## Stateless packet filtering + +Snabb NFV extends Security Groups with an option to operate statelessly. + +Stateless packet filtering has several advantages for NFV applications: + +1. Predictable performance under diverse traffic workloads. +2. No limits to the number of TCP/UDP sessions handled by the VM. +3. No CPU or memory overhead for maintaining a connection table. + +Stateless filtering can be enabled for a virtual machine port using the +`packetfilter=stateless` option: + +``` +neutron port-create ... + --binding:profile type=dict packetfilter=stateless +``` + +Note: When applying stateless filtering to a port that will be used to +initiate TCP connections, such as a management port, the [ephemeral port +range](http://en.wikipedia.org/wiki/Ephemeral_port) should be allowed on +ingress in order to accept return traffic. + +## L2TPv3 Softwire + +Snabb NFV supports L2TPv3 encapsulation for point-to-point +Ethernet-over-IPv6 tunnels. Packets transmitted by the virtual machine +will be encapsulated in L2TPv3 and packets received by the tunnel will be +decapsulated and then sent to the virtual machine. + +The local L2TPv3 tunnel endpoint for the virtual machine is the IPv6 +address and MAC address assigned by OpenStack (using the "zone" logic +above). The next-hop gateway address is configured as a template address +(e.g. `::1`) and then automatically moved into the same subnet as the +virtual machine. The remaining configuration options are specified +directly: the remote endpoint address, session ID, and cookie values. + +Example command: + +``` +neutron port-create ... + --binding:profile type=dict \ +tunnel_type=L2TPv3,\ +l2tpv3_next_hop=2003:10:1,\ +l2tpv3_remote_ip=2003:20::1,\ +l2tpv3_session=1,\ +l2tpv3_local_cookie=00000000,\ +l2tpv3_remote_cookie=00000000 +``` diff --git a/src/program/snabbnfv/neutron2snabb/neutron2snabb.lua b/src/program/snabbnfv/neutron2snabb/neutron2snabb.lua index caf41956c7..e7de0f9957 100644 --- a/src/program/snabbnfv/neutron2snabb/neutron2snabb.lua +++ b/src/program/snabbnfv/neutron2snabb/neutron2snabb.lua @@ -3,6 +3,7 @@ module(..., package.seeall) local lib = require("core.lib") local json = require("lib.json") local usage = require("program.snabbnfv.neutron2snabb.README_inc") +local neutron2snabb_schema = require("program.snabbnfv.neutron2snabb.neutron2snabb_schema") local NULL = "\\N" @@ -13,6 +14,31 @@ function run (args) create_config(unpack(args)) end +-- The Neutron database tables that we need schema information to process. +schema_tables = { + 'ml2_network_segments', 'networks', 'ports', 'ml2_port_bindings', + 'securitygrouprules', 'securitygroupportbindings' +} + +-- The default schema below is assumed if the database snapshot does +-- not include parsable table definitions. +default_schemas = { + ml2_network_segments = {'id', 'network_id', 'network_type', + 'physical_network', 'segmentation_id'}, + networks = {'tenant_id', 'id', 'name', 'status', + 'admin_state_up', 'shared'}, + ports = {'tenant_id', 'id', 'name', 'network_id', + 'mac_address', 'admin_state_up', 'status', + 'device_id', 'device_owner'}, + ml2_port_bindings = {'port_id', 'host', 'vif_type', 'driver', 'segment', + 'vnic_type', 'vif_details', 'profile'}, + securitygrouprules = {'tenant_id', 'id', 'security_group_id', + 'remote_group_id', 'direction', 'ethertype', + 'protocol', 'port_range_min', 'port_range_max', + 'remote_ip_prefix'}, + securitygroupportbindings = {'port_id', 'security_group_id'} +} + -- Create a Snabb Switch traffic process configuration. -- -- INPUT_DIR contains the Neutron database dump. @@ -22,41 +48,53 @@ end -- -- HOSTNAME is optional and defaults to the local hostname. function create_config (input_dir, output_dir, hostname) + local ok, schema = pcall(neutron2snabb_schema.read, input_dir, schema_tables) + if not ok then + print("Warning - falling back to default schema because none found:") + print(" "..schema) + schema = default_schemas + end local hostname = hostname or gethostname() local segments = parse_csv(input_dir.."/ml2_network_segments.txt", - {'id', 'network_id', 'network_type', 'physical_network', 'segmentation_id'}, - 'network_id') + schema.ml2_network_segments, + 'id') local networks = parse_csv(input_dir.."/networks.txt", - {'tenant_id', 'id', 'name', 'status', 'admin_state_up', 'shared'}, + schema.networks, 'id') local ports = parse_csv(input_dir.."/ports.txt", - {'tenant_id', 'id', 'name', 'network_id', 'mac_address', 'admin_state_up', 'status', 'device_id', 'device_owner'}, + schema.ports, 'id') local port_bindings = parse_csv(input_dir.."/ml2_port_bindings.txt", - {'id', 'host', 'vif_type', 'driver', 'segment', 'vnic_type', 'vif_details', 'profile'}, - 'id') + schema.ml2_port_bindings, + 'port_id') local secrules = parse_csv(input_dir.."/securitygrouprules.txt", - {'tenant_id', 'id', 'security_group_id', 'remote_group_id', 'direction', 'ethertype', 'protocol', 'port_range_min', 'port_range_max', 'remote_ip_prefix'}, + schema.securitygrouprules, 'security_group_id', true) local secbindings = parse_csv(input_dir.."/securitygroupportbindings.txt", - {'port_id', 'security_group_id'}, + schema.securitygroupportbindings, 'port_id') + print("Parsing neutron db tables") -- Compile zone configurations. local zones = {} for _, port in pairs(ports) do + print("PortID: ", port.id) local binding = port_bindings[port.id] -- If the port is a 'snabb' port, lives on our host and is online -- then we compile its configuration. + print("BindingID ", binding.id, " has driver ", binding.driver) if binding.driver == "snabb" then local vif_details = json.decode(binding.vif_details) -- See /~https://github.com/SnabbCo/snabbswitch/pull/423 local profile = vif_details["binding:profile"] profile = profile or {} + print("vif_details has hostname ", vif_details.zone_host, "(we want ", hostname, ")") if vif_details.zone_host == hostname then local zone_port = vif_details.zone_port -- Each zone can have multiple port configurtions. if not zones[zone_port] then zones[zone_port] = {} end + print("admin_state_ip is ", port.admin_state_up) if port.admin_state_up ~= '0' then + print("Adding zone port '", zone_port, "' to list") -- Note: Currently we don't use `vif_details.zone_gbps' -- because its "not needed by the traffic process in the -- current implementation". @@ -171,17 +209,19 @@ end function ruletofilter (r, direction) local matches = {} -- match rules to be combined - if r.ethertype == "ipv4" then matches[#matches+1] = "ip" - elseif r.ethertype == "ipv6" then matches[#matches+1] = "ip6" + local icmp + if r.ethertype == "ipv4" then matches[#matches+1] = "ip" icmp = 'icmp' + elseif r.ethertype == "ipv6" then matches[#matches+1] = "ip6" icmp = 'icmp6' else error("unknown ethertype: " .. r.ethertype) end if r.protocol == "tcp" then matches[#matches+1] = "tcp" - elseif r.protocol == "udp" then matches[#matches+1] = "udp" end - + elseif r.protocol == "udp" then matches[#matches+1] = "udp" + elseif r.protocol == "icmp" then matches[#matches+1] = icmp end + if r.port_range_min or r.port_range_max then local min = r.port_range_min or r.port_range_max local max = r.port_range_max or r.port_range_min - matches[#matches+1] = ("portrange %d-%d"):format(min, max) + matches[#matches+1] = ("dst portrange %d-%d"):format(min, max) end if r.remote_ip_prefix then @@ -220,14 +260,13 @@ function selftest () checkrule("{{direction='ingress', ethertype='IPv4', protocol='udp'}}", '(arp or (ip and udp))') checkrule("{{direction='ingress', ethertype='IPv4', protocol='udp', port_range_min=1000}}", - '(arp or (ip and udp and portrange 1000-1000))') + '(arp or (ip and udp and dst portrange 1000-1000))') checkrule("{{direction='ingress', ethertype='IPv4', protocol='udp', port_range_max=2000}}", - '(arp or (ip and udp and portrange 2000-2000))') + '(arp or (ip and udp and dst portrange 2000-2000))') checkrule("{{direction='ingress', ethertype='IPv4', protocol='tcp', port_range_min=1000, port_range_max=2000}}", - '(arp or (ip and tcp and portrange 1000-2000))') + '(arp or (ip and tcp and dst portrange 1000-2000))') checkrule("{{direction='ingress', ethertype='IPv6', protocol='tcp'}, {direction='ingress', ethertype='IPv4', protocol='udp', remote_ip_prefix='10.0.0.0/8'}}", '((ip6 and tcp) or (arp or (ip and udp and src net 10.0.0.0/8)))') print("selftest ok") end - diff --git a/src/program/snabbnfv/neutron2snabb/neutron2snabb_schema.lua b/src/program/snabbnfv/neutron2snabb/neutron2snabb_schema.lua new file mode 100644 index 0000000000..479af2f4cf --- /dev/null +++ b/src/program/snabbnfv/neutron2snabb/neutron2snabb_schema.lua @@ -0,0 +1,61 @@ +-- neutron2snabb_schema: Scan mysqldump SQL files for schema informaion +module(..., package.seeall) + +local lib = require("core.lib") + +function read (directory, tables) + local schema = {} + for _, t in ipairs(tables) do + schema[t] = columns(("%s/%s.sql"):format(directory, t)) + end + return schema +end + +-- Scan the order of keys from a table definition. + + +-- Return the columns of the named file in an array. +-- +-- The array may containing extraneous trailing items but must give +-- the actual columns in order. +function columns (filename) + -- Array of columns. + local columns = {} + local sql = lib.readfile(filename, '*a') + local definition = sql:match("CREATE TABLE `[^`]*` (%b())") + assert(definition, "failed to find CREATE TABLE definition") + -- The expected table definition format is: + -- + -- CREATE TABLE `ml2_port_bindings` ( + -- `port_id` varchar(36) NOT NULL, + -- `host` varchar(255) NOT NULL, + -- ... + -- ) ... + -- + -- We scan this and pick up the `identifiers`. + definition:gsub("`([^`]*)`", function (id) + table.insert(columns, id) + end) + return columns +end + +function selftest () + print("selftest: neutron2snabb_schema") + local neutron2snabb = require("program.snabbnfv.neutron2snabb.neutron2snabb") + -- Check that the schema we extract from the test database is + -- compaible with the default schema. (That is expected for this + -- particular data set.) + local dir = "program/snabbnfv/test_fixtures/neutron_csv" + local schema = read(dir, neutron2snabb.schema_tables) + for tab, cols in pairs(neutron2snabb.default_schemas) do + assert(schema[tab], "missing schema table: " .. tab) + for i, col in ipairs(cols) do + if schema[tab][i] ~= col then + error(("Column mismatch: %s[%d] is %s (expected %s)"):format( + tab, i, schema[tab][i], col)) + end + end + end + print("selftest: ok") +end + diff --git a/src/program/snabbnfv/neutron2snabb/selftest.sh b/src/program/snabbnfv/neutron2snabb/selftest.sh new file mode 100755 index 0000000000..bfd2ae1110 --- /dev/null +++ b/src/program/snabbnfv/neutron2snabb/selftest.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +echo "selftest: neutron2snabb/selftest.sh" + +set -e + +export TESTDIR=/tmp/snabbtest + +# Database 1 + +mkdir -p $TESTDIR + +./snabb snabbnfv neutron2snabb \ + program/snabbnfv/test_fixtures/neutron_csv $TESTDIR cdn1 + +diff $TESTDIR/port0 program/snabbnfv/test_fixtures/nfvconfig/reference/port0 +diff $TESTDIR/port2 program/snabbnfv/test_fixtures/nfvconfig/reference/port2 +echo "File contents as expected." + +echo "selftest: ok" + +rm -r $TESTDIR + diff --git a/src/program/snabbnfv/neutron_sync_agent/README b/src/program/snabbnfv/neutron_sync_agent/README index d9600a3248..c55bc7a212 100644 --- a/src/program/snabbnfv/neutron_sync_agent/README +++ b/src/program/snabbnfv/neutron_sync_agent/README @@ -12,6 +12,10 @@ point to the location of the neutron2snabb command. The default is -h HOST, --sync-host HOST Connect to snabbnfv-sync-master on HOST. Default: $SYNC_HOST + + -P PORT, --sync-port PORT + Connect to snabbnfv-sync-master on PORT. + Default: $SYNC_PORT or 9418 -p PATH, --sync-path PATH Use PATH on snabbnfv-sync-master. Default: $SYNC_PATH @@ -21,5 +25,8 @@ point to the location of the neutron2snabb command. The default is -i SECONDS, --interval SECONDS Sleep for SECONDS between sync requests. Default: $SYNC_INTERVAL or 1 + -t DIR, --tmp-dir DIR + Store temporary files in DIR. + Default: $TMP_DIR or /tmp/snabbswitch -h, --help Print this help message and exit. diff --git a/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.lua b/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.lua index ff0af3011b..de456aee9a 100644 --- a/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.lua +++ b/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.lua @@ -8,7 +8,9 @@ local script = require("program.snabbnfv.neutron_sync_agent.neutron_sync_agent_s local long_opts = { ["neutron-dir"] = "d", ["snabb-dir"] = "s", + ["tmp-dir"] = "t", ["sync-host"] = "h", + ["sync-port"] = "P", ["sync-path"] = "p", ["interval"] = "i", ["help"] = "H" @@ -18,19 +20,23 @@ function run (args) local conf = { ["NEUTRON_DIR"] = os.getenv("NEUTRON_DIR"), ["SNABB_DIR"] = os.getenv("SNABB_DIR"), + ["TMP_DIR"] = os.getenv("TMP_DIR"), ["NEUTRON2SNABB"] = os.getenv("NEUTRON2SNABB"), ["SYNC_HOST"] = os.getenv("SYNC_HOST"), + ["SYNC_PORT"] = os.getenv("SYNC_PORT"), ["SYNC_PATH"] = os.getenv("SYNC_PATH"), ["SYNC_INTERVAL"] = os.getenv("SYNC_INTERVAL") } local opt = {} function opt.d (arg) conf["NEUTRON_DIR"] = arg end function opt.s (arg) conf["SNABB_DIR"] = arg end + function opt.t (arg) conf["TMP_DIR"] = arg end function opt.h (arg) conf["SYNC_HOST"] = arg end + function opt.P (arg) conf["SYNC_PORT"] = arg end function opt.p (arg) conf["SYNC_PATH"] = arg end function opt.i (arg) conf["SYNC_INTERVAL"] = arg end function opt.H (arg) print(usage) main.exit(1) end - args = lib.dogetopt(args, opt, "d:s:h:p:i:H", long_opts) + args = lib.dogetopt(args, opt, "d:s:t:h:P:p:i:H", long_opts) local env = {} for key, value in pairs(conf) do table.insert(env, key.."="..value) diff --git a/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.sh b/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.sh index 9e6415e9ac..11dbf24cd4 100644 --- a/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.sh +++ b/src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.sh @@ -13,6 +13,7 @@ function error() { [ ! -z "$NEUTRON2SNABB" ] || export NEUTRON2SNABB="snabb snabbnfv neutron2snabb" [ ! -z "$SYNC_PATH" ] || error "check_env_vars: \$SYNC_PATH not set" [ ! -z "$SYNC_HOST" ] || error "check_env_vars: \$SYNC_HOST not set" +[ ! -z "$SYNC_PORT" ] || export SYNC_PORT=9418 [ ! -z "$SYNC_INTERVAL" ] || export SYNC_INTERVAL=1 # Remove old repository if it exists @@ -32,26 +33,34 @@ mkdir -p $TMP_DIR initial=true +function log { echo "[$(date +"%F %T %Z")]" "$1"; } + # Loop pulling/cloning the repo. while true; do if [ ! -d $NEUTRON_DIR ]; then - git clone git://$SYNC_HOST/$SYNC_PATH $NEUTRON_DIR + log "Initializing $NEUTRON_DIR" + git clone git://$SYNC_HOST:$SYNC_PORT/$SYNC_PATH $NEUTRON_DIR \ + >/dev/null 2>&1 fi - cd $NEUTRON_DIR - git fetch - git diff --quiet origin/master - if [ $? != 0 -o $initial = true ]; then - git pull --rebase origin master - git reflog expire --expire-unreachable=0 --all - git prune --expire 0 - echo "Generating new configuration" - $NEUTRON2SNABB $NEUTRON_DIR $TMP_DIR - # Only (atomically) replace configurations that have changed. - for conf in $TMP_DIR/*; do - dest=$SNABB_DIR/$(basename $conf) - if ! diff $conf $dest; then mv -f $conf $dest; fi - done - initial=false + if [ -d $NEUTRON_DIR ]; then + cd $NEUTRON_DIR + git fetch >/dev/null 2>&1 || log "Failed to fetch from master." + git diff --quiet origin/master >/dev/null 2>&1 + if [ $? = 1 -o $initial = true ]; then + log "Generating new configuration" + git pull --rebase origin master >/dev/null 2>&1 + git reflog expire --expire-unreachable=0 --all + git prune --expire 0 + $NEUTRON2SNABB $NEUTRON_DIR $TMP_DIR + # Only (atomically) replace configurations that have changed. + for conf in $(ls $TMP_DIR); do + dest=$SNABB_DIR/$(basename $conf) + echo "$dest" + if ! diff $TMP_DIR/$conf $dest; then mv -f $TMP_DIR/$conf $dest; + else echo "Unchanged."; fi + done + initial=false + fi fi sleep $SYNC_INTERVAL done diff --git a/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.lua b/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.lua index 804697fc44..207c0a9a0c 100644 --- a/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.lua +++ b/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.lua @@ -27,7 +27,7 @@ function run (args) ["DB_HOST"] = os.getenv("DB_HOST") or "localhost", ["DB_PORT"] = os.getenv("DB_PORT") or "3306", ["DB_NEUTRON"] = os.getenv("DB_NEUTRON") or "neutron_ml2", - ["DB_NEUTRON_TABLES"] = os.getenv("DB_NEUTRON_TABLES") or "networks ports ml2_network_segments securitygroups securitygrouprules securitygroupportbindings", + ["DB_NEUTRON_TABLES"] = os.getenv("DB_NEUTRON_TABLES") or "networks ports ml2_network_segments ml2_port_bindings securitygroups securitygrouprules securitygroupportbindings", ["SYNC_LISTEN_HOST"] = os.getenv("SYNC_LISTEN_HOST") or "127.0.0.1", ["SYNC_LISTEN_PORT"] = os.getenv("SYNC_LISTEN_PORT") or "9418", ["SYNC_INTERVAL"] = os.getenv("SYNC_INTERVAL") or "1" diff --git a/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.sh b/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.sh index 0289895880..3071c26f78 100644 --- a/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.sh +++ b/src/program/snabbnfv/neutron_sync_master/neutron_sync_master.sh @@ -31,32 +31,37 @@ function check_deps() (which mysqldump > /dev/null) || error "missing dependency: mysqldump" } +function log { echo "[$(date +"%F %T %Z")]" "$1"; } function run() { cd "$DB_DUMP_PATH" [ -f /tmp/neutron-sync-master.pid ] && kill $(cat /tmp/neutron-sync-master.pid) - echo "DBG: Running git daemon" export GIT_AUTHOR_NAME="Snabb NFV sync master" export GIT_AUTHOR_EMAIL="snabbnfv-sync-master" export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" + log "Starting Git daemon" git daemon --reuseaddr --listen="$SYNC_LISTEN_HOST" \ --port="$SYNC_LISTEN_PORT" --base-path="$DB_DUMP_PATH/.." --export-all \ - --verbose --pid-file=/tmp/neutron-sync-master.pid --detach "$DB_DUMP_PATH" + --verbose --pid-file=/tmp/neutron-sync-master.pid --detach "$DB_DUMP_PATH" \ + >/dev/null 2>&1 while true do - mysqldump -n -t -y -q -u${DB_USER} -p${DB_PASSWORD} -h ${DB_HOST} \ - -P ${DB_PORT} -T ${DB_DUMP_PATH} ${DB_NEUTRON} ${DB_NEUTRON_TABLES} - rm -f *.sql - git add *.txt >/dev/null - if [ $initial = true ]; then - git commit -m "Configuration update" >/dev/null - initial=false - else - git commit --amend -m "Configuration update" >/dev/null - git reflog expire --expire-unreachable=0 --all - git prune --expire 0 + mysqldump -n -y -q -u${DB_USER} -p${DB_PASSWORD} -h ${DB_HOST} \ + -P ${DB_PORT} -T ${DB_DUMP_PATH} --skip-dump-date \ + ${DB_NEUTRON} ${DB_NEUTRON_TABLES} + git add *.txt *.sql >/dev/null 2>&1 + if ! git diff --quiet --cached; then + log "Pushing configuration changes." + if [ $initial = true ]; then + git commit -m "Configuration update" >/dev/null + initial=false + else + git commit --amend -m "Configuration update" >/dev/null + git reflog expire --expire-unreachable=0 --all >/dev/null + git prune --expire 0 >/dev/null + fi fi sleep "$SYNC_INTERVAL" #check that the daemon is still running diff --git a/src/program/snabbnfv/test_fixtures/neutron_csv/ml2_network_segments.sql b/src/program/snabbnfv/test_fixtures/neutron_csv/ml2_network_segments.sql new file mode 100644 index 0000000000..7799197a3b --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/neutron_csv/ml2_network_segments.sql @@ -0,0 +1,43 @@ +-- MySQL dump 10.13 Distrib 5.5.40, for debian-linux-gnu (x86_64) +-- +-- Host: nbn-l8-dev2-os-8 Database: neutron_ml2 +-- ------------------------------------------------------ +-- Server version 5.5.40-0ubuntu0.14.04.1-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `ml2_network_segments` +-- + +DROP TABLE IF EXISTS `ml2_network_segments`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `ml2_network_segments` ( + `id` varchar(36) NOT NULL, + `network_id` varchar(36) NOT NULL, + `network_type` varchar(32) NOT NULL, + `physical_network` varchar(64) DEFAULT NULL, + `segmentation_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `network_id` (`network_id`), + CONSTRAINT `ml2_network_segments_ibfk_1` FOREIGN KEY (`network_id`) REFERENCES `networks` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed diff --git a/src/program/snabbnfv/test_fixtures/neutron_csv/ml2_port_bindings.sql b/src/program/snabbnfv/test_fixtures/neutron_csv/ml2_port_bindings.sql new file mode 100644 index 0000000000..78e08e223e --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/neutron_csv/ml2_port_bindings.sql @@ -0,0 +1,47 @@ +-- MySQL dump 10.13 Distrib 5.5.40, for debian-linux-gnu (x86_64) +-- +-- Host: nbn-l8-dev2-os-8 Database: neutron_ml2 +-- ------------------------------------------------------ +-- Server version 5.5.40-0ubuntu0.14.04.1-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `ml2_port_bindings` +-- + +DROP TABLE IF EXISTS `ml2_port_bindings`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `ml2_port_bindings` ( + `port_id` varchar(36) NOT NULL, + `host` varchar(255) NOT NULL, + `vif_type` varchar(64) NOT NULL, + `driver` varchar(64) DEFAULT NULL, + `segment` varchar(36) DEFAULT NULL, + `vnic_type` varchar(64) NOT NULL DEFAULT 'normal', + `vif_details` varchar(4095) NOT NULL DEFAULT '', + `profile` varchar(4095) NOT NULL DEFAULT '', + PRIMARY KEY (`port_id`), + KEY `segment` (`segment`), + CONSTRAINT `ml2_port_bindings_ibfk_1` FOREIGN KEY (`port_id`) REFERENCES `ports` (`id`) ON DELETE CASCADE, + CONSTRAINT `ml2_port_bindings_ibfk_2` FOREIGN KEY (`segment`) REFERENCES `ml2_network_segments` (`id`) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed diff --git a/src/program/snabbnfv/test_fixtures/neutron_csv/networks.sql b/src/program/snabbnfv/test_fixtures/neutron_csv/networks.sql new file mode 100644 index 0000000000..ab866565c9 --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/neutron_csv/networks.sql @@ -0,0 +1,42 @@ +-- MySQL dump 10.13 Distrib 5.5.40, for debian-linux-gnu (x86_64) +-- +-- Host: nbn-l8-dev2-os-8 Database: neutron_ml2 +-- ------------------------------------------------------ +-- Server version 5.5.40-0ubuntu0.14.04.1-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `networks` +-- + +DROP TABLE IF EXISTS `networks`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `networks` ( + `tenant_id` varchar(255) DEFAULT NULL, + `id` varchar(36) NOT NULL, + `name` varchar(255) DEFAULT NULL, + `status` varchar(16) DEFAULT NULL, + `admin_state_up` tinyint(1) DEFAULT NULL, + `shared` tinyint(1) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed diff --git a/src/program/snabbnfv/test_fixtures/neutron_csv/ports.sql b/src/program/snabbnfv/test_fixtures/neutron_csv/ports.sql new file mode 100644 index 0000000000..20c7cbcf74 --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/neutron_csv/ports.sql @@ -0,0 +1,47 @@ +-- MySQL dump 10.13 Distrib 5.5.40, for debian-linux-gnu (x86_64) +-- +-- Host: nbn-l8-dev2-os-8 Database: neutron_ml2 +-- ------------------------------------------------------ +-- Server version 5.5.40-0ubuntu0.14.04.1-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `ports` +-- + +DROP TABLE IF EXISTS `ports`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `ports` ( + `tenant_id` varchar(255) DEFAULT NULL, + `id` varchar(36) NOT NULL, + `name` varchar(255) DEFAULT NULL, + `network_id` varchar(36) NOT NULL, + `mac_address` varchar(32) NOT NULL, + `admin_state_up` tinyint(1) NOT NULL, + `status` varchar(16) NOT NULL, + `device_id` varchar(255) NOT NULL, + `device_owner` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `network_id` (`network_id`), + CONSTRAINT `ports_ibfk_1` FOREIGN KEY (`network_id`) REFERENCES `networks` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed diff --git a/src/program/snabbnfv/test_fixtures/neutron_csv/securitygroupportbindings.sql b/src/program/snabbnfv/test_fixtures/neutron_csv/securitygroupportbindings.sql new file mode 100644 index 0000000000..983028708e --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/neutron_csv/securitygroupportbindings.sql @@ -0,0 +1,41 @@ +-- MySQL dump 10.13 Distrib 5.5.40, for debian-linux-gnu (x86_64) +-- +-- Host: nbn-l8-dev2-os-8 Database: neutron_ml2 +-- ------------------------------------------------------ +-- Server version 5.5.40-0ubuntu0.14.04.1-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `securitygroupportbindings` +-- + +DROP TABLE IF EXISTS `securitygroupportbindings`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `securitygroupportbindings` ( + `port_id` varchar(36) NOT NULL, + `security_group_id` varchar(36) NOT NULL, + PRIMARY KEY (`port_id`,`security_group_id`), + KEY `security_group_id` (`security_group_id`), + CONSTRAINT `securitygroupportbindings_ibfk_1` FOREIGN KEY (`port_id`) REFERENCES `ports` (`id`) ON DELETE CASCADE, + CONSTRAINT `securitygroupportbindings_ibfk_2` FOREIGN KEY (`security_group_id`) REFERENCES `securitygroups` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed diff --git a/src/program/snabbnfv/test_fixtures/neutron_csv/securitygrouprules.sql b/src/program/snabbnfv/test_fixtures/neutron_csv/securitygrouprules.sql new file mode 100644 index 0000000000..798a1878f1 --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/neutron_csv/securitygrouprules.sql @@ -0,0 +1,50 @@ +-- MySQL dump 10.13 Distrib 5.5.40, for debian-linux-gnu (x86_64) +-- +-- Host: nbn-l8-dev2-os-8 Database: neutron_ml2 +-- ------------------------------------------------------ +-- Server version 5.5.40-0ubuntu0.14.04.1-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `securitygrouprules` +-- + +DROP TABLE IF EXISTS `securitygrouprules`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `securitygrouprules` ( + `tenant_id` varchar(255) DEFAULT NULL, + `id` varchar(36) NOT NULL, + `security_group_id` varchar(36) NOT NULL, + `remote_group_id` varchar(36) DEFAULT NULL, + `direction` enum('ingress','egress') DEFAULT NULL, + `ethertype` varchar(40) DEFAULT NULL, + `protocol` varchar(40) DEFAULT NULL, + `port_range_min` int(11) DEFAULT NULL, + `port_range_max` int(11) DEFAULT NULL, + `remote_ip_prefix` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `security_group_id` (`security_group_id`), + KEY `remote_group_id` (`remote_group_id`), + CONSTRAINT `securitygrouprules_ibfk_1` FOREIGN KEY (`security_group_id`) REFERENCES `securitygroups` (`id`) ON DELETE CASCADE, + CONSTRAINT `securitygrouprules_ibfk_2` FOREIGN KEY (`remote_group_id`) REFERENCES `securitygroups` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed diff --git a/src/program/snabbnfv/traffic/traffic.lua b/src/program/snabbnfv/traffic/traffic.lua index 8e49c8870a..5bda3b4586 100644 --- a/src/program/snabbnfv/traffic/traffic.lua +++ b/src/program/snabbnfv/traffic/traffic.lua @@ -6,6 +6,7 @@ local usage = require("program.snabbnfv.traffic.README_inc") local ffi = require("ffi") local C = ffi.C local timer = require("core.timer") +local pci = require("lib.hardware.pci") local long_opts = { benchmark = "B", @@ -31,6 +32,15 @@ function run (args) args = lib.dogetopt(args, opt, "hHB:k:l:D:", long_opts) if #args == 3 then local pciaddr, confpath, sockpath = unpack(args) + local ok, info = pcall(pci.device_info, pciaddr) + if not ok then + print("Error: device not found " .. pciaddr) + os.exit(1) + end + if not info.driver then + print("Error: no driver for device " .. pciaddr) + os.exit(1) + end if loadreportinterval > 0 then local t = timer.new("nfvloadreport", engine.report_load, loadreportinterval*1e9, 'repeating') timer.activate(t)