Merge PR #1167 (v2017.07 release) into master

.gitignore

@@ -20,3 +20,4 @@ __pycache__
 /src/programs.inc
 .images
 /lib/luajit/usr
+/src/core/version.lua

.version

@@ -0,0 +1 @@
+2017.07

generate-version-lua.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+cd $(dirname $0)
+
+if test -z "$VERSION"; then
+    VERSION=`cat .version`;
+fi
+
+if test -z "$EXTRAVERSION" -a -d .git; then
+    EXTRAVERSION=`git describe --dirty --tags --always --match="v$VERSION"`
+    if test "$EXTRAVERSION" = "v$VERSION"; then
+        EXTRAVERSION=
+    else
+        EXTRAVERSION=`echo "$EXTRAVERSION" | sed -e "s/^v$VERSION-//"`
+    fi
+fi
+
+cat <<EOF
+-- This file is automatically generated.
+module(..., package.seeall);
+
+version = '$VERSION'
+extra_version = '$EXTRAVERSION'
+EOF

src/Makefile

@@ -40,6 +40,8 @@ INCOBJ := $(patsubst %.inc,obj/%_inc.o, $(INCSRC))
 YANGOBJ:= $(patsubst %.yang,obj/%_yang.o, $(YANGSRC))
 EXE    := bin/snabb $(patsubst %,bin/%,$(PROGRAM))
 
+LUAJIT_A := ../lib/luajit/src/libluajit.a
+
 # TESTMODS expands to:
 #   core.memory core.lib ...
 # for each module that has a top-level selftest () function.
@@ -55,9 +57,13 @@ TESTSCRIPTS = $(shell find . -name "selftest.sh" -executable | xargs)
 PATH := ../lib/luajit/usr/local/bin:$(PATH)
 
 snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(PFLUAASMOBJ) $(INCOBJ) $(YANGOBJ) $(LUAJIT_A)
+	$(E) "GEN       obj/version.lua.gen"
+	$(Q) ../generate-version-lua.sh > obj/version.lua.gen
+	$(E) "LUA       obj/version.lua"
+	$(Q) luajit -bg -n core.version obj/version.lua.gen obj/version_lua.o
 	$(E) "LINK      $@"
 	$(Q) $(CC) $(DEBUG) -Wl,--no-as-needed -Wl,-E -Werror -Wall -o $@ $^ \
-	    ../lib/luajit/src/libluajit.a \
+	    obj/version_lua.o \
 	    -lrt -lc -ldl -lm -lpthread
 	@echo -n "BINARY    "
 	@ls -sh snabb
@@ -73,7 +79,6 @@ $(EXE): snabb bin
 	$(Q) upx -f --brute -o$@ snabb
 	@echo -n "BINARY    "
 	@ls -sh $@
-
 markdown: $(RMOBJS)
 
 test: $(TESTMODS) $(TESTSCRIPTS)

src/README.md

@@ -469,8 +469,16 @@ of this process:
 - Fully qualified: `/1234/foo/bar` ⇒ `/var/run/snabb/1234/foo/bar`
 
 Behind the scenes the objects are backed by files on ram disk
-(`/var/run/snabb/<pid>`) and accessed with the equivalent of POSIX shared
-memory (`shm_overview(7)`).
+(`/var/run/snabb/<pid>`) and accessed with the equivalent of POSIX
+shared memory (`shm_overview(7)`). The files are automatically removed
+on shutdown unless the environment `SNABB_SHM_KEEP` is set. The
+location `/var/run/snabb` can be overridden by the environment
+variable `SNABB_SHM_ROOT`.
+
+Shared memory objects are created world-readable for convenient access
+by diagnostic tools. You can lock this down by setting
+`SNABB_SHM_ROOT` to a path under a directory with appropriate
+permissions.
 
 The practical limit on the number of objects that can be mapped will depend on
 the operating system limit for memory mappings. On Linux the default limit is
@@ -808,9 +816,36 @@ commas. Example:
 comma_value(1000000) => "1,000,000"
 ```
 
-— Function **lib.random_data** *length*
+— Function **lib.random_bytes_from_dev_urandom** *length*
+
+Return *length* bytes of random data, as a byte array, taken from
+`/dev/urandom`.  Suitable for cryptographic usage.
+
+— Function **lib.random_bytes_from_math_random** *length*
+
+Return *length* bytes of random data, as a byte array, where each byte
+was taken from `math.random(0, 255)`.  *Not* suitable for cryptographic
+usage.
+
+— Function **lib.random_bytes** *length*
+— Function **lib.randomseed** *seed*
 
-Returns a string of *length* bytes of random data.
+Initialize Snabb's random number generation facility.  If *seed* is nil,
+then the Lua `math.random()` function will be seeded from
+`/dev/urandom`, and `lib.random_bytes` will be initialized to
+`lib.random_bytes_from_dev_urandom`.  This is Snabb's default mode of
+operation.
+
+Sometimes it's useful to make Snabb use deterministic random numbers.
+In that case, pass a seed to **lib.randomseed**; Snabb will set
+`lib.random_bytes` to `lib.random_bytes_from_math_random`, and also
+print out a message to stderr indicating that we are using lower-quality
+deterministic random numbers.
+
+As part of its initialization process, Snabb will call `lib.randomseed`
+with the value of the `SNABB_RANDOM_SEED` environment variable (if
+any).  Set this environment variable to enable deterministic random
+numbers.
 
 — Function **lib.bounds_checked** *type*, *base*, *offset*, *size*
 
@@ -890,6 +925,11 @@ integers *n* respectively. Unsigned.
 Network to host byte order conversion functions for 32 and 16 bit
 integers *n* respectively. Unsigned.
 
+— Function **lib.random_bytes** *count*
+
+Return a fresh array of *count* random bytes.  Suitable for
+cryptographic usage.
+
 — Function **lib.parse** *arg*, *config*
 
 Validates *arg* against the specification in *config*, and returns a fresh

src/apps/bridge/mac_table.lua

@@ -553,7 +553,6 @@ function selftest ()
       local info = t:info()
       assert(info.size == s)
       assert(info.mask == info.buckets-1)
-      math.randomseed(0)
       local macs = {}
       local n = 0
       while true do

src/apps/config/action_codec.lua

@@ -98,12 +98,9 @@ local extra = "0123456789_-"
 local alphabet = table.concat({lower_case, upper_case, extra})
 assert(#alphabet == 64)
 local function random_file_name()
-   local f = io.open('/dev/urandom', 'rb')
    -- 22 bytes, but we only use 2^6=64 bits from each byte, so total of
    -- 132 bits of entropy.
-   local bytes = f:read(22)
-   assert(#bytes == 22)
-   f:close()
+   local bytes = lib.random_data(22)
    local out = {}
    for i=1,#bytes do
       table.insert(out, alphabet:byte(bytes:byte(i) % 64 + 1))

src/apps/intel_mp/intel_mp.lua

@@ -255,7 +255,6 @@ Intel = {
       txq = {},
       rxq = {},
       mtu = {default=9014},
-      rssseed = {default=314159},
       linkup_wait = {default=120},
       wait_for_link = {default=false},
       master_stats = {default=true},
@@ -289,7 +288,6 @@ function Intel:new (conf)
       txq = conf.txq,
       rxq = conf.rxq,
       mtu = conf.mtu or self.config.mtu.default,
-      rssseed = conf.rssseed or self.config.mtu.default,
       linkup_wait = conf.linkup_wait or self.config.linkup_wait.default,
       wait_for_link = conf.wait_for_link
    }
@@ -554,7 +552,6 @@ function Intel:rss_enable ()
    self:rss_key()
 end
 function Intel:rss_key ()
-   math.randomseed(self.rssseed)
    for i=0,9,1 do
       self.r.RSSRK[i](math.random(2^32))
    end

src/apps/lwaftr/ctable_wrapper.lua

@@ -40,21 +40,16 @@ end
 function new(params)
    local ctab = ctable.new(params)
    ctab.add_with_random_ejection = add_with_random_ejection
-   -- Not local-ized because it's called once
-   math.randomseed(bxor(os.time(), S.getpid()))
    return ctab
 end
 
 function selftest()
    local ffi = require("ffi")
-   local hash_32 = ctable.hash_32
-
    local occupancy = 4
    -- 32-byte entries 
    local params = {
       key_type = ffi.typeof('uint32_t'),
       value_type = ffi.typeof('int32_t[6]'),
-      hash_fn = hash_32,
       max_occupancy_rate = 0.4,
       initial_size = ceil(occupancy / 0.4)
    }

src/apps/lwaftr/fragmentv4_hardened.lua

@@ -33,7 +33,6 @@ constants.o_ipv4_flags, constants.o_ipv4_checksum,
 constants.o_ipv4_total_length, constants.o_ipv4_src_addr,
 constants.o_ipv4_dst_addr
 
-local hash_32 = ctable.hash_32
 local rd16, wr16, wr32 = lwutil.rd16, lwutil.wr16, lwutil.wr32
 local get_ihl_from_offset = lwutil.get_ihl_from_offset
 local uint16_ptr_t = ffi.typeof("uint16_t*")
@@ -232,20 +231,6 @@ local function packet_to_reassembly_buffer(pkt)
    return reassembly_buf
 end
 
--- The key is 80 bits: source IPv4 address, destination IPv4 address, and
--- the 16-bit identification field.
--- This function intentionally re-hashes 3 of the 5 16-byte chunks.
-local function hash_ipv4(key)
-   local hash = 0
-   local to_hash = ffi.cast(uint16_ptr_t, key)
-   for i=0,3 do
-      local current = to_hash[i]
-      hash = hash_32(bxor(hash, hash_32(current)))
-   end
-
-   return hash
-end
-
 function initialize_frag_table(max_fragmented_packets, max_pkt_frag)
    -- Initialize module-scoped variables
    max_frags_per_packet = max_pkt_frag
@@ -268,7 +253,6 @@ function initialize_frag_table(max_fragmented_packets, max_pkt_frag)
    local params = {
       key_type = ffi.typeof(ipv4_fragment_key_t),
       value_type = ffi.typeof(ipv4_reassembly_buffer_t),
-      hash_fn = hash_ipv4,
       initial_size = math.ceil(max_fragmented_packets / max_occupy),
       max_occupancy_rate = max_occupy,
    }

src/apps/lwaftr/fragmentv6_hardened.lua

@@ -37,7 +37,6 @@ constants.o_ipv6_frag_offset, constants.o_ipv6_frag_id,
 constants.o_ipv6_payload_len, constants.ipv6_frag_header_size,
 constants.o_ipv6_next_header
 
-local hash_32 = ctable.hash_32
 local rd16, rd32 = lwutil.rd16, lwutil.rd32
 local uint32_ptr_t = ffi.typeof("uint32_t*")
 local bxor, band = bit.bxor, bit.band
@@ -226,18 +225,6 @@ local function packet_to_reassembly_buffer(pkt)
    return reassembly_buf
 end
 
--- The key is 288 bits: source IPv6 address, destination IPv6 address, and
--- the identification field from the IPv6 fragmentation header.
-local function hash_ipv6(key)
-   local hash = 0
-   local to_hash = ffi.cast(uint32_ptr_t, key)
-   for i=0,8 do
-      local current = to_hash[i]
-      hash = hash_32(bxor(hash, hash_32(current)))
-   end
-   return hash
-end
-
 function initialize_frag_table(max_fragmented_packets, max_pkt_frag)
    -- Initialize module-scoped variables
    max_frags_per_packet = max_pkt_frag
@@ -260,7 +247,6 @@ function initialize_frag_table(max_fragmented_packets, max_pkt_frag)
    local params = {
       key_type = ffi.typeof(ipv6_fragment_key_t),
       value_type = ffi.typeof(ipv6_reassembly_buffer_t),
-      hash_fn = hash_ipv6,
       initial_size = math.ceil(max_fragmented_packets / max_occupy),
       max_occupancy_rate = max_occupy,
    }