Merge PR #981 (v2016.08 release) into master

src/README.md

@@ -104,6 +104,13 @@ the engine for use in processing and are *read-only*.
 Name of the app. *Read-only*.
 
 
+— Field **myapp.shm**
+
+Can be set to a specification for `core.shm.create_frame` during `new`. When
+set, this field will be initialized to a frame of shared memory objects by the
+engine.
+
+
 — Method **myapp:link**
 
 *Optional*. Called any time the app’s links may have been changed (including on
@@ -401,7 +408,7 @@ can be accessed directly by network cards. The important
 characteristic of DMA memory is being located in contiguous physical
 memory at a stable address.
 
-— Function **memory.dma_alloc** *bytes*, *[alignment]*
+— Function **memory.dma_alloc** *bytes*, [*alignment*]
 
 Returns a pointer to *bytes* of new DMA memory.
 
@@ -417,6 +424,210 @@ Returns the physical address (`uint64_t`) the DMA memory at *pointer*.
 Size of a single huge page in bytes. Read-only.
 
 
+## Shared Memory (core.shm)
+
+This module facilitates creation and management of named shared memory objects.
+Objects can be created using `shm.create` similar to `ffi.new`, except that
+separate calls to `shm.open` for the same name will each return a new mapping
+of the same shared memory. Different processes can share memory by mapping an
+object with the same name (and type). Each process can map any object any
+number of times.
+
+Mappings are deleted on process termination or with an explicit `shm.unmap`.
+Names are unlinked from objects that are no longer needed using `shm.unlink`.
+Object memory is freed when the name is unlinked and all mappings have been
+deleted.
+
+Names can be fully qualified or abbreviated to be within the current process.
+Here are examples of names and how they are resolved where `<pid>` is the PID
+of this process:
+
+- Local: `foo/bar` ⇒ `/var/run/snabb/<pid>/foo/bar`
+- Fully qualified: `/1234/foo/bar` ⇒ `/var/run/snabb/1234/foo/bar`
+
+Behind the scenes the objects are backed by files on ram disk
+(`/var/run/snabb/<pid>`) and accessed with the equivalent of POSIX shared
+memory (`shm_overview(7)`).
+
+The practical limit on the number of objects that can be mapped will depend on
+the operating system limit for memory mappings. On Linux the default limit is
+65,530 mappings:
+
+```
+$ sysctl vm.max_map_count vm.max_map_count = 65530
+```
+
+— Function **shm.create** *name*, *type*
+
+Creates and maps a shared object of *type* into memory via a hierarchical
+*name*. Returns a pointer to the mapped object.
+
+— Function **shm.open** *name*, *type*, [*readonly*]
+
+Maps an existing shared object of *type* into memory via a hierarchical *name*.
+If *readonly* is non-nil the shared object is mapped in read-only mode.
+*Readonly* defaults to nil. Fails if the shared object does not already exist.
+Returns a pointer to the mapped object.
+
+— Function **shm.unmap** *pointer*
+
+Deletes the memory mapping for *pointer*.
+
+— Function **shm.unlink** *path*
+
+Unlinks the subtree of objects designated by *path* from the filesystem.
+
+— Function **shm.children** *path*
+
+Returns an array of objects in the directory designated by *path*.
+
+— Function **shm.register** *type*, *module*
+
+Registers an abstract shared memory object *type* implemented by *module* in
+`shm.types`. *Module* must provide the following functions:
+
+ - **create** *name*, ...
+ - **open**, *name*
+
+and can optionally provide the function:
+
+ - **delete**, *name*
+
+The *module*’s `type` variable must be bound to *type*. To register a new type
+a module might invoke `shm.register` like so:
+
+```
+type = shm.register('mytype', getfenv())
+-- Now the following holds true:
+--   shm.types[type] == getfenv()
+```
+
+— Variable **shm.types**
+
+A table that maps types to modules. See `shm.register`.
+
+— Function **shm.create_frame** *path*, *specification*
+
+Creates and returns a shared memory frame by *specification* under *path*. A
+frame is a table of mapped—possibly abstract‑shared memory objects.
+*Specification* must be of the form:
+
+```
+{ <name> = {<module>, ...},
+  ... }
+```
+
+*Module* must implement an abstract type registered with `shm.register`, and is
+followed by additional initialization arguments to its `create` function.
+Example usage:
+
+```
+local counter = require("core.counter")
+-- Create counters foo/bar/{dtime,rxpackets,txpackets}.counter
+local f = shm.create_frame(
+   "foo/bar",
+   {dtime     = {counter, C.get_unix_time()},
+    rxpackets = {counter},
+    txpackets = {counter}})
+counter.add(f.rxpackets)
+counter.read(f.dtime)
+```
+
+— Function **shm.open_frame** *path*
+
+Opens and returns the shared memory frame under *path* for reading.
+
+— Function **shm.delete_frame** *frame*
+
+Deletes/unmaps a shared memory *frame*. The *frame* directory is unlinked if
+*frame* was created by `shm.create_frame`.
+
+
+### Counter (core.counter)
+
+Double-buffered shared memory counters. Counters are 64-bit unsigned values.
+Registered with `core.shm` as type `counter`.
+
+— Function **counter.create** *name*, [*initval*]
+
+Creates and returns a `counter` by *name*, initialized to *initval*. *Initval*
+defaults to 0.
+
+— Function **counter.open** *name*
+
+Opens and returns the counter by *name* for reading.
+
+— Function **counter.delete** *name*
+
+Deletes and unmaps the counter by *name*.
+
+— Function **counter.commit**
+
+Commits buffered counter values to public shared memory.
+
+— Function **counter.set** *counter*, *value*
+
+Sets *counter* to *value*.
+
+— Function **counter.add** *counter*, [*value*]
+
+Increments *counter* by *value*. *Value* defaults to 1.
+
+— Function **counter.read** *counter*
+
+Returns the value of *counter*.
+
+
+### Histogram (core.histogram)
+
+Shared memory histogram with logarithmic buckets. Registered with `core.shm` as
+type `histogram`.
+
+— Function **histogram.new** *min*, *max*
+
+Returns a new `histogram`, with buckets covering the range from *min* to *max*.
+The range between *min* and *max* will be divided logarithmically.
+
+— Function **histogram.create** *name*, *min*, *max*
+
+Creates and returns a `histogram` as in `histogram.new` by *name*. If the file
+exists already, it will be cleared.
+
+— Function **histogram.open** *name*
+
+Opens and returns `histogram` by *name* for reading.
+
+— Method **histogram:add** *measurement*
+
+Adds *measurement* to *histogram*.
+
+— Method **histogram:iterate** *prev*
+
+When used as `for count, lo, hi in histogram:iterate()`, visits all buckets in
+*histogram* in order from lowest to highest. *Count* is the number of samples
+recorded in that bucket, and *lo* and *hi* are the lower and upper bounds of
+the bucket. Note that *count* is an unsigned 64-bit integer; to get it as a Lua
+number, use `tonumber`.
+
+If *prev* is given, it should be a snapshot of the previous version of the
+histogram. In that case, the *count* values will be returned as a difference
+between their values in *histogram* and their values in *prev*.
+
+— Method **histogram:snapshot** [*dest*]
+
+Copies out the contents of *histogram* into the `histogram` *dest* and returns
+*dest*. If *dest* is not given, the result will be a fresh `histogram`.
+
+— Method **histogram:clear**
+
+Clears the buckets of *histogram*.
+
+— Method **histogram:wrap_thunk* *thunk*, *now*
+
+Returns a closure that wraps *thunk*, measuring and recording the difference
+between calls to *now* before and after *thunk* into *histogram*.
+
+
 ## Lib (core.lib)
 
 The `core.lib` module contains miscellaneous utilities.
@@ -674,4 +885,4 @@ A list of command-line arguments to the running script. Read-only.
 
 — Function **main.exit** *status*
 
-Cleanly exists the process with *status*.
+Cleanly exits the process with *status*.

src/apps/bridge/flooding.lua

@@ -21,20 +21,18 @@ function bridge:new (arg)
 end
 
 function bridge:push()
-   local src_ports = self._src_ports
+   local ports = self._ports
    local dst_ports = self._dst_ports
-   local output = self.output
    local i = 1
-   while src_ports[i] do
-      local src_port = src_ports[i]
-      local l_in = self.input[src_port]
+   while ports[i] do
+      local l_in = ports[i].l_in
       while not empty(l_in) do
-         local ports = dst_ports[src_port]
+         local dst = dst_ports[i]
          local p = receive(l_in)
-         transmit(output[ports[1]], p)
+         transmit(ports[dst[1]].l_out, p)
          local j = 2
-         while ports[j] do
-            transmit(output[ports[j]], clone(p))
+         while dst[j] do
+            transmit(ports[dst[j]].l_out, clone(p))
             j = j + 1
          end
       end

src/apps/intel/intel1g.lua

@@ -632,7 +632,7 @@ end  -- function Intel1g:new()
 
 function selftest ()
    print("selftest: Intel1g")
-   local pciaddr = os.getenv("SNABB_PCI_INTEL1G0")
+   local pciaddr = lib.getenv("SNABB_PCI_INTEL1G0")
    if not pciaddr then
       print("SNABB_PCI_INTEL1G0 not set")
       os.exit(engine.test_skipped_code)

src/apps/intel/intel_app.lua

@@ -34,12 +34,6 @@ local function firsthole(t)
    end
 end
 
-local provided_counters = {
-   'type', 'dtime', 'mtu', 'speed', 'status', 'promisc', 'macaddr',
-   'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', 'rxerrors',
-   'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop', 'txerrors'
-}
-
 -- Create an Intel82599 App for the device with 'pciaddress'.
 function Intel82599:new (arg)
    local conf = config.parse_app_arg(arg)
@@ -63,21 +57,31 @@ function Intel82599:new (arg)
       self.stats = { s = self.dev.s, r = self.dev.r, qs = self.dev.qs }
       self.zone = "intel"
    end
-   if not self.stats.counters then
-      self.stats.path = "/counters/"..conf.pciaddr.."/"
+   if not self.stats.shm then
+      self.stats.shm = shm.create_frame(
+         "pci/"..conf.pciaddr,
+         {dtime     = {counter, C.get_unix_time()},
+          mtu       = {counter, self.dev.mtu},
+          speed     = {counter, 10000000000}, -- 10 Gbits
+          status    = {counter, 2},           -- Link down
+          promisc   = {counter},
+          macaddr   = {counter},
+          rxbytes   = {counter},
+          rxpackets = {counter},
+          rxmcast   = {counter},
+          rxbcast   = {counter},
+          rxdrop    = {counter},
+          rxerrors  = {counter},
+          txbytes   = {counter},
+          txpackets = {counter},
+          txmcast   = {counter},
+          txbcast   = {counter},
+          txdrop    = {counter},
+          txerrors  = {counter}})
       self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now)
-      self.stats.counters = {}
-      for _, name in ipairs(provided_counters) do
-         self.stats.counters[name] = counter.open(self.stats.path..name)
-      end
-      counter.set(self.stats.counters.type, 0x1000) -- Hardware interface
-      counter.set(self.stats.counters.dtime, C.get_unix_time())
-      counter.set(self.stats.counters.mtu, self.dev.mtu)
-      counter.set(self.stats.counters.speed, 10000000000) -- 10 Gbits
-      counter.set(self.stats.counters.status, 2) -- down
+
       if not conf.vmdq and conf.macaddr then
-         counter.set(self.stats.counters.macaddr,
-                     macaddress:new(conf.macaddr).bits)
+         counter.set(self.stats.shm.macaddr, macaddress:new(conf.macaddr).bits)
       end
    end
    return setmetatable(self, Intel82599)
@@ -102,10 +106,7 @@ function Intel82599:stop()
       close_pf:close()
    end
    if not self.dev.pf or close_pf then
-      for name, _ in pairs(self.stats.counters) do
-         counter.delete(self.stats.path..name)
-      end
-      shm.unlink(self.stats.path)
+      shm.delete_frame(self.stats.shm)
    end
 end
 
@@ -117,7 +118,7 @@ function Intel82599:reconfig(arg)
    self.dev:reconfig(conf)
 
    if not self.dev.pf and conf.macaddr then
-      counter.set(self.stats.counters.macaddr,
+      counter.set(self.stats.shm.macaddr,
                   macaddress:new(conf.macaddr).bits)
    end
 end
@@ -153,11 +154,11 @@ function Intel82599:add_receive_buffers ()
    end
 end
 
--- Synchronize self.stats.s/r a and self.stats.counters.
+-- Synchronize self.stats.s/r a and self.stats.shm.
 local link_up_mask = lib.bits{Link_up=30}
 local promisc_mask = lib.bits{UPE=9}
 function Intel82599:sync_stats ()
-   local counters = self.stats.counters
+   local counters = self.stats.shm
    local s, r, qs = self.stats.s, self.stats.r, self.stats.qs
    counter.set(counters.rxbytes,   s.GORC64())
    counter.set(counters.rxpackets, s.GPRC())
@@ -195,7 +196,7 @@ function Intel82599:push ()
       -- check is currently disabled to satisfy some selftests until
       -- agreement on this strategy is reached.
       -- if p.length > self.dev.mtu then
-      --    counter.add(self.stats.counters.txdrop)
+      --    counter.add(self.stats.shm.txdrop)
       --    packet.free(p)
       -- else
       do local p = receive(l)