Merge PR #1322 (v2018.04 release) into master

.version

@@ -1 +1 @@
-2018.01
+2018.04

Dockerfile

@@ -0,0 +1,13 @@
+FROM alpine:3.7 AS build
+RUN apk add --no-cache libgcc alpine-sdk gcc libpcap-dev linux-headers findutils
+COPY . /snabb
+RUN cd /snabb && make clean && make -j && cd src && make -j
+
+FROM alpine:3.7
+RUN apk add --no-cache libgcc
+COPY --from=build /snabb/src/snabb /usr/local/bin/
+
+VOLUME /u
+WORKDIR /u
+
+ENTRYPOINT ["/usr/local/bin/snabb"]

Makefile

@@ -47,4 +47,9 @@ dist: all
 	cd "$(DISTDIR)/.." && tar cJvf "`basename '$(DISTDIR)'`.tar.xz" "`basename '$(DISTDIR)'`"
 	rm -rf "$(DISTDIR)"
 
+docker:
+	docker build -t snabb .
+	@ln -sf ../src/scripts/dock.sh src/snabb
+	@echo "Usage: docker run -ti --rm snabb <program> ..."
+	@echo "or simply call 'src/snabb <program> ...'"
 .SERIAL: all

README.md

@@ -114,12 +114,33 @@ $ cp src/snabb /usr/local/bin/
 $ sudo snabb packetblaster replay capture.pcap 01:00.0
 ```
 
+### snabb container
+
+Basic support for building and running snabb in a Docker container is available via
+
+```
+$ make docker
+```
+
+This will build a tiny snabb container (8MB), ready to be used:
+
+```
+$ docker run -ti --rm snabb --help
+```
+
+Or simply run snabb, as you would under linux. This is made possible by using a wrapper shell script that
+gets linked to as part of 'make docker':
+
+```
+$ src/snabb --help
+```
+
 ## How do I get involved?
 
 Here are the ways you can get involved:
 
 - Use the Snabb applications in your network.
 - Create your very own application: [Getting Started](src/doc/getting-started.md).
 - Create Github Issues with your ideas and questions and problems.
-- Hang out on the [Snabb Slack chat](https://snabb.slack.com/). You can get a no-questions-asked invitation by mailing `luke@snabb.co` with the addresses/domains you want invited.
+- [Join](https://join.slack.com/t/snabb/shared_invite/enQtMzIyOTIwMTg5ODYyLTMzY2FjMGEzM2QzNDlhMDYxNzU0M2UyNjQ1MDc4MDRjY2Q3MWMwY2Q4YWQ1NDllY2E3NTZkZGUyZTQxNzgyNjc) the [Snabb Slack chat](https://snabb.slack.com/) to hang out and shoot the breeze.
 

lib/ljsyscall/syscall/bsd/syscalls.lua

@@ -31,6 +31,7 @@ if C.getdirentries then
     basep = basep or t.long1()
     local ret, err = C.getdirentries(getfd(fd), buf, size, basep)
     if ret == -1 then return nil, t.error(err or errno()) end
+    if ret == 0 then return nil, nil end
     return t.dirents(buf, ret)
   end
 end

lib/ljsyscall/syscall/syscalls.lua

@@ -740,6 +740,7 @@ if C.getdents then
     buf = buf or t.buffer(size)
     local ret, err = C.getdents(getfd(fd), buf, size)
     if ret == -1 then return nil, t.error(err or errno()) end
+    if ret == 0 then return nil, nil end
     return t.dirents(buf, ret)
   end
 end

lib/ljsyscall/syscall/types.lua

@@ -601,15 +601,10 @@ if bsdtypes then types = bsdtypes.init(c, types) end
 -- define dents type if dirent is defined
 if t.dirent then
   t.dirents = function(buf, size) -- buf should be char*
-    local d, i = nil, 0
+    local i = 0
     return function() -- TODO work out if possible to make stateless
-      if size > 0 and not d then
-        d = pt.dirent(buf)
-        i = i + d.d_reclen
-        return d
-      end
       while i < size do
-        d = pt.dirent(pt.char(d) + d.d_reclen)
+        local d = pt.dirent(buf + i)
         i = i + d.d_reclen
         if d.ino ~= 0 then return d end -- some systems use ino = 0 for deleted files before removed eg OSX; it is never valid
       end

lib/ljsyscall/syscall/util.lua

@@ -56,22 +56,19 @@ function util.ls(name, buf, size)
   if err then return nil, err end
   local di
   return function()
-    local d, first
-    repeat
+    while true do
+      if di then
+        local d = di()
+        if d then return d.name, d end
+      end
+      -- Fetch more entries.
+      local err
+      di, err = fd:getdents(buf, size)
       if not di then
-        local err
-        di, err = fd:getdents(buf, size)
-        if not di then
-          fd:close()
-          error(err)
-        end
-        first = true
+        fd:close()
+        if err then error(err) else return nil end
       end
-      d = di()
-      if not d then di = nil end
-      if not d and first then return nil end
-    until d
-    return d.name, d
+    end
   end
 end
 

src/README.md

@@ -725,10 +725,6 @@ Returns the filename of the first file in *directory*.
 
 Returns the first line of file at *filename* as a string.
 
-— Function **lib.files_in_directory** *directory*
-
-Returns an array of filenames in *directory*.
-
 — Function **lib.load_string** *string*
 
 Evaluates and returns the value of the Lua expression in *string*.

src/apps/intel/intel_app.lua

@@ -252,14 +252,10 @@ end
 function selftest ()
    print("selftest: intel_app")
 
-   local pcideva = lib.getenv("SNABB_PCI_INTEL0") or lib.getenv("SNABB_PCI0")
-   local pcidevb = lib.getenv("SNABB_PCI_INTEL1") or lib.getenv("SNABB_PCI1")
-   if not pcideva
-      or pci.device_info(pcideva).driver ~= 'apps.intel.intel_app'
-      or not pcidevb
-      or pci.device_info(pcidevb).driver ~= 'apps.intel.intel_app'
-   then
-      print("SNABB_PCI_INTEL[0|1]/SNABB_PCI[0|1] not set or not suitable.")
+   local pcideva = lib.getenv("SNABB_PCI_INTEL0")
+   local pcidevb = lib.getenv("SNABB_PCI_INTEL1")
+   if not pcideva or not pcidevb then
+      print("SNABB_PCI_INTEL[0|1] not set or not suitable.")
       os.exit(engine.test_skipped_code)
    end
 

src/apps/intel_mp/selftest.sh

@@ -14,6 +14,7 @@ TESTS=$(echo "$TESTS1G" "$TESTS10G" | grep -e "$FILTER" | sort)
 ESTATUS=0
 export SNABB_RECV_DEBUG=true
 export SNABB_RECV_MASTER_STATS=true
+export SNABB_RANDOM_SEED=0xacabba9e
 for i in $TESTS; do
    pkill -P $$ -f snabb
    sleep 1

src/apps/ipsec/selftest.sh

@@ -3,7 +3,7 @@
 
 SKIPPED_CODE=43
 
-if [ "$SNABB_IPSEC_SKIP_E2E_TEST" = yes ]; then
+if [ -z "$SNABB_IPSEC_ENABLE_E2E_TEST" ]; then
     exit $SKIPPED_CODE
 fi
 

src/apps/ipv4/README.md

@@ -53,7 +53,7 @@ traffic will go to a single MAC address.  If this address is provided
 as part of the configuration, no ARP request will be made; otherwise
 it will be determined from the *next_ip* via ARP.
 
-— Key **self_ip**
+— Key **next_ip**
 
 *Optional*.  The IPv4 address of the next-hop host.  Required only if
  *next_mac* is not specified as part of the configuration.

src/core/app.lua

@@ -505,9 +505,9 @@ function pace_breathing ()
       else
          sleep = math.floor(sleep/2)
       end
-      lastfrees = counter.read(frees)
-      lastfreebytes = counter.read(freebytes)
-      lastfreebits = counter.read(freebits)
+      lastfrees = tonumber(counter.read(frees))
+      lastfreebytes = tonumber(counter.read(freebytes))
+      lastfreebits = tonumber(counter.read(freebits))
    end
 end
 

src/core/lib.lua

@@ -107,14 +107,6 @@ end
 
 function firstline (filename) return readfile(filename, "*l") end
 
-function files_in_directory (dir)
-   local files = {}
-   for line in assert(io.popen('ls -1 "'..dir..'" 2>/dev/null')):lines() do
-      table.insert(files, line)
-   end
-   return files
-end
-
 -- Load Lua value from string.
 function load_string (string)
    return loadstring("return "..string)()
@@ -726,6 +718,7 @@ function random_bytes_from_dev_urandom (count)
    while written < count do
       written = written + assert(f:read(bytes, count-written))
    end
+   f:close()
    return bytes
 end
 

src/core/main.lua

@@ -225,23 +225,23 @@ end
 -- Fork a child process that monitors us and performs cleanup actions
 -- when we terminate.
 local snabbpid = S.getpid()
+local lockfile = os.tmpname()
+local lock = S.open(lockfile, "wronly")
+S.unlink(lockfile)
+S.sigprocmask("block", "hup, int, quit, term")
+lock:lockf("lock", 0)
 if assert(S.fork()) ~= 0 then
-   -- parent process: run snabb
+   -- Parent process; run Snabb.
+   S.sigprocmask("unblock", "hup, int, quit, term")
    xpcall(main, handler)
+   -- Lock will be released however the process exits.
 else
-   -- child process: supervise parent & perform cleanup
-   -- Subscribe to SIGHUP on parent death
+   -- Child process: Supervise parent & perform cleanup.  Lock not
+   -- inherited from parent.
    S.prctl("set_name", "[snabb sup]")
-   S.prctl("set_pdeathsig", "hup")
-   -- Trap relevant signals to a file descriptor
-   local exit_signals = "hup, int, quit, term"
-   local signalfd = S.signalfd(exit_signals)
-   S.sigprocmask("block", exit_signals)
-   -- wait until we receive a signal
-   local signals
-   repeat signals = assert(S.util.signalfd_read(signalfd)) until #signals > 0
-   -- cleanup after parent process
+   -- Wait for parent to release lock.
+   lock:lockf("lock", 0)
+   -- Finally, clean up after parent process.
    shutdown(snabbpid)
-   -- exit with signal-appropriate status
-   os.exit(128 + signals[1].signo)
+   os.exit(128)
 end

src/doc/snabblab.md

@@ -11,15 +11,16 @@ Want to be a known developer? Sure! Just edit the [user account list](https://gi
 
 ## Servers
 
-Name        | Purpose                                           | SSH                     | Xeon model   | NICs
+Name        | Purpose                                           | SSH                     | Intel CPU    | NICs
 ------------|---------------------------------------------------|-------------------------| --------     | ------------------------------------------------
-lugano-1    | General use                                       | lugano-1.snabb.co       | E3 1650v3    | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710)
-lugano-2    | General use                                       | lugano-2.snabb.co       | E3 1650v3    | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710)
-lugano-3    | General use                                       | lugano-3.snabb.co       | E3 1650v3    | 2 x 10G (82599), 2 x 100G (ConnectX-4)
-lugano-4    | General use                                       | lugano-4.snabb.co       | E3 1650v3    | 2 x 10G (82599), 2 x 100G (ConnectX-4)
+lugano-1    | General use                                       | lugano-1.snabb.co       | E5 1650v3    | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710)
+lugano-2    | General use                                       | lugano-2.snabb.co       | E5 1650v3    | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710)
+lugano-3    | General use                                       | lugano-3.snabb.co       | E5 1650v3    | 2 x 10G (82599), 2 x 100G (ConnectX-4)
+lugano-4    | General use                                       | lugano-4.snabb.co       | E5 1650v3    | 2 x 10G (82599), 2 x 100G (ConnectX-4)
 davos       | Continuous Integration tests & driver development | lab1.snabb.co port 2000 | 2x E5 2603   | Diverse 10G/40G: Intel, SolarFlare, Mellanox, Chelsio, Broadcom. Installed upon request.
 grindelwald | Snabb NFV testing                                 | lab1.snabb.co port 2010 | 2x E5 2697v2 | 12 x 10G (Intel 82599)
 interlaken  | Haswell/AVX2 testing                              | lab1.snabb.co port 2030 | 2x E5 2620v3 | 12 x 10G (Intel 82599)
+murren-*    | Hydra fleet for tests without NICs                | (none)                  | i7-6700      | (none)
 
 ## Get started
 

src/lib/ctable.lua

@@ -7,7 +7,7 @@ local lib = require("core.lib")
 local binary_search = require("lib.binary_search")
 local multi_copy = require("lib.multi_copy")
 local siphash = require("lib.hash.siphash")
-local max, floor, ceil = math.max, math.floor, math.ceil
+local min, max, floor, ceil = math.min, math.max, math.floor, math.ceil
 
 CTable = {}
 LookupStreamer = {}
@@ -154,6 +154,7 @@ end
 local try_huge_pages = true
 local huge_page_threshold = 1e6
 local function calloc(t, count)
+   if count == 0 then return 0, 0 end
    local byte_size = ffi.sizeof(t) * count
    local mem, err
    if try_huge_pages and byte_size > huge_page_threshold then
@@ -204,6 +205,7 @@ end
 
 function CTable:resize(size)
    assert(size >= (self.occupancy / self.max_occupancy_rate))
+   assert(size == floor(size))
    local old_entries = self.entries
    local old_size = self.size
    local old_max_displacement = self.max_displacement
@@ -280,7 +282,7 @@ function CTable:add(key, value, updates_allowed)
    if self.occupancy + 1 > self.occupancy_hi then
       -- Note that resizing will invalidate all hash keys, so we need
       -- to hash the key after resizing.
-      self:resize(self.size * 2)
+      self:resize(max(self.size * 2, 1)) -- Could be current size is 0.
    end
 
    local hash = self.hash_fn(key)
@@ -402,7 +404,7 @@ function CTable:remove_ptr(entry)
    end
 
    if self.occupancy < self.occupancy_lo then
-      self:resize(self.size / 2)
+      self:resize(max(ceil(self.size / 2), 1))
    end
 end
 
@@ -589,7 +591,7 @@ function CTable:next_entry(offset, limit)
    elseif limit == nil then
       limit = self.size + self.max_displacement
    else
-      limit = math.min(limit, self.size + self.max_displacement)
+      limit = min(limit, self.size + self.max_displacement)
    end
    for offset=offset, limit-1 do
       if self.entries[offset].hash ~= HASH_MAX then
@@ -612,7 +614,6 @@ function selftest()
       initial_size = ceil(occupancy / 0.4)
    }
    local ctab = new(params)
-   ctab:resize(occupancy / 0.4 + 1)
 
    -- Fill with {i} -> { bnot(i), ... }.
    local k = ffi.new('uint32_t[1]');
@@ -682,7 +683,8 @@ function selftest()
          -- keep references to avoid GCing too early
          local handle = {}
          local function read(size)
-            local buf = ffi.new('uint8_t[?]', size, file:read(size))
+            local buf = ffi.new('uint8_t[?]', size)
+            ffi.copy(buf, file:read(size), size)
             table.insert(handle, buf)
             return buf
          end
@@ -706,7 +708,7 @@ function selftest()
    repeat
       local streamer = ctab:make_lookup_streamer(width)
       for i = 1, occupancy, width do
-         local n = math.min(width, occupancy-i+1)
+         local n = min(width, occupancy-i+1)
          for j = 0, n-1 do
             streamer.entries[j].key[0] = i + j
          end

src/lib/hardware/pci.lua

@@ -27,9 +27,11 @@ devices = {}
 
 --- Initialize (or re-initialize) the `devices` table.
 function scan_devices ()
-   for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do
-      local info = device_info(device)
-      if info.driver then table.insert(devices, info) end
+   for device in assert(S.util.ls("/sys/bus/pci/devices")) do
+      if device ~= '.' and device ~= '..' then
+         local info = device_info(device)
+         if info.driver then table.insert(devices, info) end
+      end
    end
 end