Initial commit

.gitignore

@@ -0,0 +1,6 @@
+/_build
+/deps
+/.dialyzer
+erl_crash.dump
+*.ez
+doc

LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2017 Blitz Studios, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

bench/snowflake.exs

@@ -0,0 +1,9 @@
+Application.ensure_all_started(:snowflake)
+
+# To test against snowflakex, uncomment the second line in the benchmark
+# and also add snowflakex as a dependency to your mix.exs
+
+Benchee.run(%{
+  "snowflake" => fn -> Snowflake.next_id() end,
+  # "snowflakex" => fn -> Snowflakex.new() end
+})

changelog.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## v 0.0.1
+
+Initial Release

config/config.exs

@@ -0,0 +1,5 @@
+use Mix.Config
+
+config :snowflake,
+  nodes: ["127.0.0.1"],
+  epoch: 1473562509301

lib/snowflake.ex

@@ -0,0 +1,26 @@
+defmodule Snowflake do
+  @moduledoc """
+  Generates Snowflake IDs
+  """
+  use Application
+
+  def start(_type, _args) do
+    import Supervisor.Spec
+
+    children = [
+      worker(Snowflake.Generator, [Snowflake.Utils.epoch(), Snowflake.Utils.machine_id()])
+    ]
+
+    Supervisor.start_link(children, strategy: :one_for_one)
+  end
+
+  @doc """
+  Generates a snowflake ID, each call is guaranteed to return a different ID
+  that is sequantially larger than the previous ID.
+  """
+  @spec next_id() :: {:ok, integer} |
+                     {:error, :backwards_clock}
+  def next_id() do
+    GenServer.call(Snowflake.Generator, :next_id)
+  end
+end

lib/snowflake/generator.ex

@@ -0,0 +1,53 @@
+defmodule Snowflake.Generator do
+  @moduledoc false
+  use GenServer
+
+  @machine_id_overflow 1024
+  @seq_overflow 4096
+
+  def start_link(epoch, machine_id) when machine_id < @machine_id_overflow do
+    state = {epoch, ts(epoch), machine_id, 0}
+    GenServer.start_link(__MODULE__, state, name: __MODULE__)
+  end
+
+  def handle_call(:next_id, from, {epoch, prev_ts, machine_id, seq} = state) do
+    case next_ts_and_seq(epoch, prev_ts, seq) do
+      {:error, :seq_overflow} ->
+        :timer.sleep(1)
+        handle_call(:next_id, from, state)
+      {:error, :backwards_clock} ->
+        {:reply, {:error, :backwards_clock}, state}
+      {:ok, new_ts, new_seq} ->
+        new_state = {epoch, new_ts, machine_id, new_seq}
+        {:reply, {:ok, create_id(new_ts, machine_id, new_seq)}, new_state}
+    end
+  end
+
+  defp next_ts_and_seq(epoch, prev_ts, seq) do
+    case ts(epoch) do
+      ^prev_ts ->
+        case seq + 1 do
+          @seq_overflow -> {:error, :seq_overflow}
+          next_seq -> {:ok, prev_ts, next_seq}
+        end
+      new_ts ->
+        cond do
+          new_ts < prev_ts -> {:error, :backwards_clock}
+          true -> {:ok, new_ts, 0}
+        end
+    end
+  end
+
+  defp create_id(ts, machine_id, seq) do
+    << new_id :: unsigned-integer-size(64)>> = <<
+       ts :: unsigned-integer-size(42),
+       machine_id :: unsigned-integer-size(10),
+       seq :: unsigned-integer-size(12) >>
+
+    new_id
+  end
+
+  defp ts(epoch) do
+    System.os_time(:milliseconds) - epoch
+  end
+end

lib/snowflake/helper.ex

@@ -0,0 +1,42 @@
+defmodule Snowflake.Helper do
+  @moduledoc """
+  The Helper module helps users work with snowflake IDs.
+
+  Helper module can do the following:
+  - Deriving timestamp based on ID
+  - Creating buckets based on days since epoch...
+  """
+  use Bitwise
+
+  @doc """
+  Get timestamp in ms from epoch from any snowflake ID
+  """
+  @spec timestamp_of_id(integer) :: integer
+  def timestamp_of_id(id) do
+    id >>> 22
+  end
+
+  @doc """
+  Get bucket value based on segments of N days
+  """
+  @spec bucket(integer, atom, integer) :: integer
+  def bucket(units, unit_type, id) do
+    round(timestamp_of_id(id) / bucket_size(unit_type, units))
+  end
+
+  @doc """
+  When no id is provided, we generate a bucket for the current time
+  """
+  @spec bucket(integer, atom) :: integer
+  def bucket(units, unit_type) do
+    timestamp = System.os_time(:milliseconds) - Snowflake.Utils.epoch()
+    round(timestamp / bucket_size(unit_type, units))
+  end
+
+  defp bucket_size(unit_type, units) do
+    case unit_type do
+      :hours -> 1000 * 60 * 60 * units
+      _ -> 1000 * 60 * 60 * 24 * units  # days is default
+    end
+  end
+end

lib/snowflake/utils.ex

@@ -0,0 +1,58 @@
+defmodule Snowflake.Utils do
+  @moduledoc """
+  Utility functions intended for Snowflake application.
+  epoch() and machine_id() are useful for inspecting in production.
+  """
+
+  @doc """
+  Grabs epoch from config value
+  """
+  @spec epoch() :: integer
+  def epoch() do
+    Application.get_env(:snowflake, :epoch)
+  end
+
+  @doc """
+  Grabs hostname, fqdn, and ip addresses, then compares that list to the nodes
+  config to find the intersection.
+  """
+  @spec machine_id() :: integer
+  def machine_id() do
+    nodes = Application.get_env(:snowflake, :nodes)
+    host_addrs = [hostname(), fqdn()] ++ ip_addrs()
+
+    case MapSet.intersection(MapSet.new(host_addrs), MapSet.new(nodes)) |> Enum.take(1) do
+      [matching_node] -> Enum.find_index(nodes, fn node -> node == matching_node end)
+      _ -> 1023
+    end
+  end
+
+  defp ip_addrs() do
+    case :inet.getifaddrs do
+      {:ok, ifaddrs} ->
+        ifaddrs
+        |> Enum.flat_map(fn {_, kwlist} ->
+          kwlist |> Enum.filter(fn {type, _} -> type == :addr end)
+        end)
+        |> Enum.filter_map(fn {_, addr} -> tuple_size(addr) in [4, 6] end, fn {_, addr} ->
+          case addr do
+            {a, b, c, d} -> [a, b, c, d] |> Enum.join(".")              # ipv4
+            {a, b, c, d, e, f} -> [a, b, c, d, e, f] |> Enum.join(":")  # ipv6
+          end
+        end)
+      _ -> []
+    end
+  end
+
+  defp hostname() do
+    {:ok, name} = :inet.gethostname()
+    to_string(name)
+  end
+
+  defp fqdn() do
+    case :inet.get_rc[:domain] do
+      nil -> nil
+      domain -> hostname() <> "." <> to_string(domain)
+    end
+  end
+end

mix.exs

@@ -0,0 +1,53 @@
+defmodule Snowflake.Mixfile do
+  use Mix.Project
+
+  @version "0.0.1"
+  @url "https://github.com/blitzstudios/snowflake"
+  @maintainers ["Weixi Yen"]
+
+  def project do
+    [
+      name: "Snowflake",
+      app: :snowflake,
+      version: @version,
+      source_url: @url,
+      build_embedded: Mix.env == :prod,
+      start_permanent: Mix.env == :prod,
+      maintainers: @maintainers,
+      description: "Elixir Snowflake ID Generator",
+      elixir: "~> 1.3",
+      package: package(),
+      homepage_url: @url,
+      docs: docs(),
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [applications: [],
+     mod: {Snowflake, []}]
+  end
+
+  defp deps do
+    [
+      {:dialyxir, "~> 0.4", only: :dev, runtime: false},
+      {:benchee, "~> 0.6", only: :dev}
+    ]
+  end
+
+  def docs do
+    [
+      extras: ["README.md", "CHANGELOG.md"],
+      source_ref: "v#{@version}"
+    ]
+  end
+
+  defp package do
+    [
+      maintainers: @maintainers,
+      licenses: ["MIT"],
+      links: %{github: @url},
+      files: ~w(lib) ++ ~w(CHANGELOG.md LICENSE mix.exs README.md)
+    ]
+  end
+end

mix.lock

@@ -0,0 +1,4 @@
+%{"benchee": {:hex, :benchee, "0.6.0", "c2565506c621ee010e71d05f555e39a1b937e00810e284bc85463a4d4efc4b00", [:mix], [{:deep_merge, "~> 0.1", [hex: :deep_merge, optional: false]}]},
+  "deep_merge": {:hex, :deep_merge, "0.1.1", "c27866a7524a337b6a039eeb8dd4f17d458fd40fbbcb8c54661b71a22fffe846", [:mix], []},
+  "dialyxir": {:hex, :dialyxir, "0.4.4", "e93ff4affc5f9e78b70dc7bec7b07da44ae1ed3fef38e7113568dd30ad7b01d3", [:mix], []},
+  "snowflakex": {:hex, :snowflakex, "1.0.0", "a621dc8004625b178f4248cf5ea412c1f83d84f89b7bdc2479f4019bd26c34d4", [:mix], []}}

readme.md

@@ -0,0 +1,89 @@
+# Snowflake
+
+A distributed Snowflake generator in Elixir.
+
+## Usage
+
+In your mix.exs file:
+
+```elixir
+def deps do
+  [{:snowflake, "~> 0.0.1"}]
+end
+```
+
+Specify the nodes in your config.  If your running a cluster, specify all the nodes in the cluster that snowflake runs on.  
+
+- **nodes** can be Public IPs, Private IPs, Hostnames, or FQDNs
+- **epoch** should not be changed once you begin generating IDs and want to maintain sorting
+- There should be no more than 1 snowflake generator per node, or you risk duplicate potential duplicate snowflakes on the same node.
+
+```elixir
+config :snowflake,
+  nodes: [127.0.0.1],   # up to 1023 nodes
+  epoch: 1142974214000  # don't change after you decide what your epoch is
+```
+
+Generating an ID is simple.
+
+```elixir
+Snowflake.next_id()
+# => 828746161129414660
+
+Snowflake.next_hundred_ids()
+# => [828746161129414661 .. 828746161129414761]
+```
+
+## Helper functions
+
+After generating snowflake IDs, you may want to use them to do other things.
+For example, deriving a bucket number from a snowflake to use as part of a
+composite key in Cassandra in the attempt to limit partition size.
+
+Lets say we want to know the current bucket for an ID that would be generated right now:
+```elixir
+Snowflake.Helper.bucket(30, :days)
+# => 5
+```
+
+Or if we want to know which bucket a snowflake ID should belong to, given we are
+bucketing by every 30 days.
+```elixir
+Snowflake.Helper.bucket(30, :days, 828746161129414661)
+# => 76
+```
+
+Or if we want to know how many ms elapsed from epoch
+```elixir
+Snowflake.Helper.timestamp_of_id(828746161129414661)
+# => 197588482172
+```
+
+## NTP
+
+Keep your nodes in sync with [ntpd](https://en.wikipedia.org/wiki/Ntpd) or use
+your VM equivalent as snowflake depends on OS time.  ntpd's job is to slow down
+or speed up the clock so that it syncs the time with the network time.
+
+## Architecture
+
+Snowflake allows the user to specify the nodes in the cluster, each representing a machine.  Snowflake at startup inspects itself for IP and Host information and derives its machine_id from the location of itself in the list of nodes defined in the config.
+
+Machine ID is defaulted to **1023** if there is no configuration or if snowflake does not find a match to remain highly available.  It is important to specify the correct IPs / Hostnames / FQDNs for the nodes in a production environment to avoid any chance of snowflake collision.
+
+## Benchmarks
+
+Consistently generates over 60,000 snowflakes per second on Macbook Pro 2.5 GHz Intel Core i7 w/ 16 GB RAM.
+
+```
+Benchmarking snowflake...
+Benchmarking snowflakex...
+
+Name                 ips        average  deviation         median
+snowflake       316.51 K        3.16 μs   ±503.52%        3.00 μs
+snowflakex      296.26 K        3.38 μs   ±514.60%        3.00 μs
+
+Comparison:
+snowflake       316.51 K
+snowflakex      296.26 K - 1.07x slower
+```