From e49ceab354a0ab70b76741b27c8f7bd907bfb3f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 14 May 2018 14:40:51 +0200
Subject: [PATCH 01/40] Introduce Firenest.PG

---
 lib/firenest/pg.ex | 150 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)
 create mode 100644 lib/firenest/pg.ex

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
new file mode 100644
index 0000000..b3900d2
--- /dev/null
+++ b/lib/firenest/pg.ex
@@ -0,0 +1,150 @@
+defmodule Firenest.PG do
+  alias Firenest.SyncedServer
+
+  @type pg() :: SyncedServer.server()
+
+  defdelegate child_spec(opts), to: Firenest.PG.Supervisor
+
+  def track(pg, pid, topic, key, meta) when node(pid) == node() do
+    server = partition_info!(pg, topic)
+    GenServer.call(server, {:track, pid, topic, key, meta})
+  end
+
+  def untrack(pg, pid, topic, key) when node(pid) == node() do
+    server = partition_info!(pg, topic)
+    GenServer.call(server, {:untrack, pid, topic, key})
+  end
+
+  def untrack(pg, pid) when node(pid) == node() do
+    servers = partition_infos!(pg)
+    multicall(servers, {:untrack, pid}, 5_000)
+  end
+
+  def update(pg, pid, topic, key, meta) when node(pid) == node() do
+    server = partition_info!(pg, topic)
+    GenServer.call(server, {:update, pid, topic, key, meta})
+  end
+
+  def list(pg, topic) do
+    server = partition_info!(pg, topic)
+    GenServer.call(server, {:list, topic}).()
+  end
+
+  # TODO
+  # def dirty_list(pg, topic)
+
+  defp multicall(pids, request, timeout) do
+    pids
+    |> Enum.map(fn pid ->
+      ref = Process.monitor(pid)
+      send(pid, {:"$gen_call", {self(), ref}, request})
+      ref
+    end)
+    |> Enum.each(fn ref ->
+      receive do
+        {^ref, reply} ->
+          Process.demonitor(ref, [:flush])
+          reply
+
+        {:DOWN, ^ref, _, _, reason} ->
+          exit(reason)
+      after
+        timeout ->
+          Process.demonitor(ref, [:flush])
+          exit(:timeout)
+      end
+    end)
+  end
+
+  defp partition_info!(pg, topic) do
+    hash = :erlang.phash2(topic)
+    extract = {:element, {:+, {:rem, {:const, hash}, :"$1"}, 1}, :"$2"}
+    ms = [{{:partitions, :"$1", :"$2"}, [], [extract]}]
+    [info] = :ets.select(pg, ms)
+    info
+  catch
+    :error, :badarg ->
+      raise ArgumentError, "unknown group: #{inspect(pg)}"
+  end
+
+  defp partition_infos!(pg) do
+    Tuple.to_list(:ets.lookup_element(pg, :partitions, 3))
+  catch
+    :error, :badarg ->
+      raise ArgumentError, "unknown group: #{inspect(pg)}"
+  end
+end
+
+defmodule Firenest.PG.Supervisor do
+  @moduledoc false
+  use Supervisor
+
+  def child_spec(opts) do
+    partitions = Keyword.get(opts, :partitions, 1)
+    name = Keyword.fetch!(opts, :name)
+    topology = Keyword.fetch!(opts, :topology)
+    supervisor = Module.concat(name, "Supervisor")
+    arg = {partitions, name, topology, opts}
+
+    %{
+      id: __MODULE__,
+      start: {Supervisor, :start_link, [__MODULE__, arg, name: supervisor]},
+      type: :supervisor
+    }
+  end
+
+  def init({partitions, name, topology, opts}) do
+    names =
+      for partition <- 0..(partitions - 1),
+          do: Module.concat(name, "Partition" <> Integer.to_string(partition))
+
+    children =
+      for name <- names,
+          do: {Firenest.PG.Server, {name, topology, opts}}
+
+    :ets.new(name, [:named_table, :set, read_concurrency: true])
+    :ets.insert(name, {:partitions, partitions, List.to_tuple(names)})
+
+    Supervisor.init(children, strategy: :one_for_one)
+  end
+end
+
+defmodule Firenest.PG.Server do
+  @moduledoc false
+  use Firenest.SyncedServer
+
+  alias Firenest.SyncedServer
+
+  def child_spec({name, topology, opts}) do
+    server_opts = [name: name, topology: topology]
+
+    %{
+      id: name,
+      start: {SyncedServer, :start_link, [__MODULE__, {name, opts}, server_opts]}
+    }
+  end
+
+  @impl true
+  def init({name, _opts}) do
+    values = :ets.new(name, [:named_table, :protected, :ordered_set])
+    pids = :ets.new(__MODULE__.Pids, [:duplicate_bag])
+    {:ok, %{values: values, pids: pids}}
+  end
+
+  @impl true
+  # def handle_call({:track, pid, topic, key, meta}, state) do
+
+  # end
+
+  # def handle_call({:untrack, pid, topic, key}, state) do
+
+  # end
+
+  # def handle_call({:untrack, pid}, state) do
+
+  # end
+
+  # def handle_call({:list, topic}, state) do
+
+  # end
+end

From aac13833716e3b3b1d66ed7e229ae61a62782765 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 14 May 2018 16:36:24 +0200
Subject: [PATCH 02/40] Remove race condition from PG.multicall

---
 lib/firenest/pg.ex | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index b3900d2..bce2c72 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -33,9 +33,10 @@ defmodule Firenest.PG do
   # TODO
   # def dirty_list(pg, topic)
 
-  defp multicall(pids, request, timeout) do
-    pids
-    |> Enum.map(fn pid ->
+  defp multicall(servers, request, timeout) do
+    servers
+    |> Enum.map(fn server ->
+      pid = Process.whereis(server)
       ref = Process.monitor(pid)
       send(pid, {:"$gen_call", {self(), ref}, request})
       ref

From 32ea17fa4cf94cb8109c6f7806766028f6e8d832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 15 May 2018 19:33:26 +0200
Subject: [PATCH 03/40] Implement basic local tracking

---
 lib/firenest/pg.ex        | 73 +++++++++++++++++++++++++++------------
 test/firenest/pg_test.exs | 46 ++++++++++++++++++++++++
 2 files changed, 96 insertions(+), 23 deletions(-)
 create mode 100644 test/firenest/pg_test.exs

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index bce2c72..6b05bb1 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -5,14 +5,14 @@ defmodule Firenest.PG do
 
   defdelegate child_spec(opts), to: Firenest.PG.Supervisor
 
-  def track(pg, pid, topic, key, meta) when node(pid) == node() do
-    server = partition_info!(pg, topic)
-    GenServer.call(server, {:track, pid, topic, key, meta})
+  def track(pg, pid, group, key, meta) when node(pid) == node() do
+    server = partition_info!(pg, group)
+    GenServer.call(server, {:track, pid, group, key, meta})
   end
 
-  def untrack(pg, pid, topic, key) when node(pid) == node() do
-    server = partition_info!(pg, topic)
-    GenServer.call(server, {:untrack, pid, topic, key})
+  def untrack(pg, pid, group, key) when node(pid) == node() do
+    server = partition_info!(pg, group)
+    GenServer.call(server, {:untrack, pid, group, key})
   end
 
   def untrack(pg, pid) when node(pid) == node() do
@@ -20,18 +20,18 @@ defmodule Firenest.PG do
     multicall(servers, {:untrack, pid}, 5_000)
   end
 
-  def update(pg, pid, topic, key, meta) when node(pid) == node() do
-    server = partition_info!(pg, topic)
-    GenServer.call(server, {:update, pid, topic, key, meta})
+  def update(pg, pid, group, key, meta) when node(pid) == node() do
+    server = partition_info!(pg, group)
+    GenServer.call(server, {:update, pid, group, key, meta})
   end
 
-  def list(pg, topic) do
-    server = partition_info!(pg, topic)
-    GenServer.call(server, {:list, topic}).()
+  def list(pg, group) do
+    server = partition_info!(pg, group)
+    GenServer.call(server, {:list, group}).()
   end
 
   # TODO
-  # def dirty_list(pg, topic)
+  # def dirty_list(pg, group)
 
   defp multicall(servers, request, timeout) do
     servers
@@ -57,8 +57,8 @@ defmodule Firenest.PG do
     end)
   end
 
-  defp partition_info!(pg, topic) do
-    hash = :erlang.phash2(topic)
+  defp partition_info!(pg, group) do
+    hash = :erlang.phash2(group)
     extract = {:element, {:+, {:rem, {:const, hash}, :"$1"}, 1}, :"$2"}
     ms = [{{:partitions, :"$1", :"$2"}, [], [extract]}]
     [info] = :ets.select(pg, ms)
@@ -89,7 +89,7 @@ defmodule Firenest.PG.Supervisor do
 
     %{
       id: __MODULE__,
-      start: {Supervisor, :start_link, [__MODULE__, arg, name: supervisor]},
+      start: {Supervisor, :start_link, [__MODULE__, arg, [name: supervisor]]},
       type: :supervisor
     }
   end
@@ -127,17 +127,22 @@ defmodule Firenest.PG.Server do
 
   @impl true
   def init({name, _opts}) do
+    Process.flag(:trap_exit, true)
     values = :ets.new(name, [:named_table, :protected, :ordered_set])
-    pids = :ets.new(__MODULE__.Pids, [:duplicate_bag])
+    pids = :ets.new(__MODULE__.Pids, [:duplicate_bag, keypos: 2])
     {:ok, %{values: values, pids: pids}}
   end
 
   @impl true
-  # def handle_call({:track, pid, topic, key, meta}, state) do
-
-  # end
+  def handle_call({:track, pid, group, key, meta}, _from, state) do
+    %{values: values, pids: pids} = state
+    Process.link(pid)
+    :ets.insert(values, {{group, pid, key}, meta})
+    :ets.insert(pids, {group, pid, key})
+    {:reply, :ok, state}
+  end
 
-  # def handle_call({:untrack, pid, topic, key}, state) do
+  # def handle_call({:untrack, pid, group, key}, state) do
 
   # end
 
@@ -145,7 +150,29 @@ defmodule Firenest.PG.Server do
 
   # end
 
-  # def handle_call({:list, topic}, state) do
+  def handle_call({:list, group}, _from, state) do
+    %{values: values} = state
 
-  # end
+    read = fn ->
+      ms = [{{{group, :_, :"$1"}, :"$2"}, [], [{{:"$1", :"$2"}}]}]
+      :ets.select(values, ms)
+    end
+
+    {:reply, read, state}
+  end
+
+  @impl true
+  def handle_info({:EXIT, pid, reason}, state) do
+    %{values: values, pids: pids} = state
+
+    case :ets.take(pids, pid) do
+      [] ->
+        {:stop, state, reason}
+
+      list when is_list(list) ->
+        ms = for key <- list, do: {{key, :_}, [], [true]}
+        :ets.select_delete(values, ms)
+        {:noreply, state}
+    end
+  end
 end
diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
new file mode 100644
index 0000000..0b9e078
--- /dev/null
+++ b/test/firenest/pg_test.exs
@@ -0,0 +1,46 @@
+defmodule Firenest.PGTest do
+  use ExUnit.Case, async: true
+
+  alias Firenest.PG
+
+  setup_all do
+    {:ok, topology: Firenest.Test, evaluator: Firenest.Test.Evaluator}
+  end
+
+  setup %{test: test, topology: topology} do
+    start_supervised!({PG, name: test, topology: topology})
+    {:ok, pg: test}
+  end
+
+  test "tracks processes", %{pg: pg} do
+    PG.track(pg, self(), :foo, :bar, :baz)
+    assert [{:bar, :baz}] == PG.list(pg, :foo)
+  end
+
+  test "processes are clened up when they die", %{pg: pg} do
+    {pid, ref} = spawn_monitor(Process, :sleep, [:infinity])
+    PG.track(pg, pid, :foo, :bar, :baz)
+    assert [_] = PG.list(pg, :foo)
+    Process.exit(pid, :kill)
+    assert_receive {:DOWN, ^ref, _, _, _}
+    assert [] = PG.list(pg, :foo)
+  end
+
+  test "pg dies if linked process terminates", %{pg: pg} do
+    parent = self()
+    [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+    ref = Process.monitor(pid)
+
+    temp =
+      spawn(fn ->
+        Process.link(pid)
+        send(parent, :continue)
+        Process.sleep(:infinity)
+      end)
+
+    assert_receive :continue
+
+    Process.exit(temp, :shutdown)
+    assert_receive {:DOWN, ^ref, _, _, _}
+  end
+end

From 87ebdc8b5682abca7b7de1c4c256ca9dd316bd98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 16 May 2018 12:57:16 +0200
Subject: [PATCH 04/40] More basic functions for local tracker

---
 lib/firenest/pg.ex        | 44 ++++++++++++++++++++++++++++++++-------
 test/firenest/pg_test.exs | 16 +++++++++++++-
 2 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index 6b05bb1..db3d910 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -142,13 +142,35 @@ defmodule Firenest.PG.Server do
     {:reply, :ok, state}
   end
 
-  # def handle_call({:untrack, pid, group, key}, state) do
+  def handle_call({:untrack, pid, group, key}, _from, state) do
+    %{values: values, pids: pids} = state
+    key = {group, pid, key}
+    ms = [{key, [], [true]}]
+
+    case :ets.select_delete(pids, ms) do
+      0 ->
+        {:reply, {:error, :not_tracked}, state}
 
-  # end
+      1 ->
+        unless :ets.member(pids, pid) do
+          Process.unlink(pid)
+        end
 
-  # def handle_call({:untrack, pid}, state) do
+        :ets.delete(values, key)
+        {:reply, :ok, state}
+    end
+  end
 
-  # end
+  def handle_call({:untrack, pid}, _from, state) do
+    %{values: values, pids: pids} = state
+
+    if untrack_pid(pids, values, pid) do
+      Process.unlink(pid)
+      {:reply, :ok, state}
+    else
+      {:reply, {:error, :not_tracked}, state}
+    end
+  end
 
   def handle_call({:list, group}, _from, state) do
     %{values: values} = state
@@ -165,14 +187,22 @@ defmodule Firenest.PG.Server do
   def handle_info({:EXIT, pid, reason}, state) do
     %{values: values, pids: pids} = state
 
+    if untrack_pid(pids, values, pid) do
+      {:noreply, state}
+    else
+      {:stop, reason, state}
+    end
+  end
+
+  defp untrack_pid(pids, values, pid) do
     case :ets.take(pids, pid) do
       [] ->
-        {:stop, state, reason}
+        false
 
-      list when is_list(list) ->
+      list ->
         ms = for key <- list, do: {{key, :_}, [], [true]}
         :ets.select_delete(values, ms)
-        {:noreply, state}
+        true
     end
   end
 end
diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
index 0b9e078..aadf911 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/pg_test.exs
@@ -26,7 +26,7 @@ defmodule Firenest.PGTest do
     assert [] = PG.list(pg, :foo)
   end
 
-  test "pg dies if linked process terminates", %{pg: pg} do
+  test "pg dies if linked, untracked process terminates", %{pg: pg} do
     parent = self()
     [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
     ref = Process.monitor(pid)
@@ -43,4 +43,18 @@ defmodule Firenest.PGTest do
     Process.exit(temp, :shutdown)
     assert_receive {:DOWN, ^ref, _, _, _}
   end
+
+  test "untrack/2", %{pg: pg} do
+    PG.track(pg, self(), :foo, :bar, :baz)
+    assert [_] = PG.list(pg, :foo)
+    PG.untrack(pg, self())
+    assert [] == PG.list(pg, :foo)
+  end
+
+  test "untrack/4", %{pg: pg} do
+    PG.track(pg, self(), :foo, :bar, :baz)
+    assert [_] = PG.list(pg, :foo)
+    PG.untrack(pg, self(), :foo, :bar)
+    assert [] == PG.list(pg, :foo)
+  end
 end

From 46051ca32faa1680f9f33906c5c78dc66bd9a444 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 16 May 2018 15:19:11 +0200
Subject: [PATCH 05/40] All local elements of PG work correctly

---
 lib/firenest/pg.ex        |  89 ++++++++++++++++++------
 test/firenest/pg_test.exs | 142 +++++++++++++++++++++++++++++---------
 2 files changed, 178 insertions(+), 53 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index db3d910..2cb40d7 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -5,29 +5,40 @@ defmodule Firenest.PG do
 
   defdelegate child_spec(opts), to: Firenest.PG.Supervisor
 
-  def track(pg, pid, group, key, meta) when node(pid) == node() do
+  def join(pg, group, key, pid, meta) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:track, pid, group, key, meta})
+    GenServer.call(server, {:join, group, key, pid, meta})
   end
 
-  def untrack(pg, pid, group, key) when node(pid) == node() do
+  def leave(pg, group, key, pid) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:untrack, pid, group, key})
+    GenServer.call(server, {:leave, group, key, pid})
   end
 
-  def untrack(pg, pid) when node(pid) == node() do
+  def leave(pg, pid) when node(pid) == node() do
     servers = partition_infos!(pg)
-    multicall(servers, {:untrack, pid}, 5_000)
+    replies = multicall(servers, {:leave, pid}, 5_000)
+
+    if :ok in replies do
+      :ok
+    else
+      {:error, :not_member}
+    end
   end
 
-  def update(pg, pid, group, key, meta) when node(pid) == node() do
+  def update(pg, group, key, pid, update) when node(pid) == node() and is_function(update, 1) do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:update, pid, group, key, meta})
+    GenServer.call(server, {:update, group, key, pid, update})
   end
 
-  def list(pg, group) do
+  def replace(pg, group, key, pid, meta) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:list, group}).()
+    GenServer.call(server, {:replace, group, key, pid, meta})
+  end
+
+  def members(pg, group) do
+    server = partition_info!(pg, group)
+    GenServer.call(server, {:members, group}).()
   end
 
   # TODO
@@ -41,7 +52,7 @@ defmodule Firenest.PG do
       send(pid, {:"$gen_call", {self(), ref}, request})
       ref
     end)
-    |> Enum.each(fn ref ->
+    |> Enum.map(fn ref ->
       receive do
         {^ref, reply} ->
           Process.demonitor(ref, [:flush])
@@ -134,22 +145,28 @@ defmodule Firenest.PG.Server do
   end
 
   @impl true
-  def handle_call({:track, pid, group, key, meta}, _from, state) do
+  def handle_call({:join, group, key, pid, meta}, _from, state) do
     %{values: values, pids: pids} = state
     Process.link(pid)
-    :ets.insert(values, {{group, pid, key}, meta})
-    :ets.insert(pids, {group, pid, key})
-    {:reply, :ok, state}
+    ets_key = {group, pid, key}
+
+    if :ets.member(values, ets_key) do
+      {:reply, {:error, :already_joined}, state}
+    else
+      :ets.insert(values, {{group, pid, key}, meta})
+      :ets.insert(pids, {group, pid, key})
+      {:reply, :ok, state}
+    end
   end
 
-  def handle_call({:untrack, pid, group, key}, _from, state) do
+  def handle_call({:leave, group, key, pid}, _from, state) do
     %{values: values, pids: pids} = state
     key = {group, pid, key}
     ms = [{key, [], [true]}]
 
     case :ets.select_delete(pids, ms) do
       0 ->
-        {:reply, {:error, :not_tracked}, state}
+        {:reply, {:error, :not_member}, state}
 
       1 ->
         unless :ets.member(pids, pid) do
@@ -161,18 +178,44 @@ defmodule Firenest.PG.Server do
     end
   end
 
-  def handle_call({:untrack, pid}, _from, state) do
+  def handle_call({:update, group, key, pid, update}, _from, state) do
+    %{values: values} = state
+    ets_key = {group, pid, key}
+
+    case ets_fetch_element(values, ets_key, 2) do
+      {:ok, value} ->
+        :ets.insert(values, {ets_key, update.(value)})
+        {:reply, :ok, state}
+
+      :error ->
+        {:reply, {:error, :not_member}, state}
+    end
+  end
+
+  def handle_call({:replace, group, key, pid, meta}, _from, state) do
+    %{values: values} = state
+    ets_key = {group, pid, key}
+
+    if :ets.member(values, ets_key) do
+      :ets.insert(values, {ets_key, meta})
+      {:reply, :ok, state}
+    else
+      {:reply, {:error, :not_member}, state}
+    end
+  end
+
+  def handle_call({:leave, pid}, _from, state) do
     %{values: values, pids: pids} = state
 
     if untrack_pid(pids, values, pid) do
       Process.unlink(pid)
       {:reply, :ok, state}
     else
-      {:reply, {:error, :not_tracked}, state}
+      {:reply, {:error, :not_member}, state}
     end
   end
 
-  def handle_call({:list, group}, _from, state) do
+  def handle_call({:members, group}, _from, state) do
     %{values: values} = state
 
     read = fn ->
@@ -205,4 +248,10 @@ defmodule Firenest.PG.Server do
         true
     end
   end
+
+  defp ets_fetch_element(table, key, pos) do
+    {:ok, :ets.lookup_element(table, key, pos)}
+  catch
+    :error, :badarg -> :error
+  end
 end
diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
index aadf911..6cb1413 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/pg_test.exs
@@ -12,49 +12,125 @@ defmodule Firenest.PGTest do
     {:ok, pg: test}
   end
 
-  test "tracks processes", %{pg: pg} do
-    PG.track(pg, self(), :foo, :bar, :baz)
-    assert [{:bar, :baz}] == PG.list(pg, :foo)
+  describe "join/5" do
+    test "adds process", %{pg: pg} do
+      assert PG.join(pg, :foo, :bar, self(), :baz) == :ok
+      assert [{:bar, :baz}] == PG.members(pg, :foo)
+    end
+
+    test "rejects double joins", %{pg: pg} do
+      assert PG.join(pg, :foo, :bar, self(), :baz) == :ok
+      assert PG.join(pg, :foo, :bar, self(), :baz) == {:error, :already_joined}
+    end
+
+    test "cleans up entries after process dies", %{pg: pg} do
+      {pid, ref} = spawn_monitor(Process, :sleep, [:infinity])
+      PG.join(pg, :foo, :bar, pid, :baz)
+      assert [_] = PG.members(pg, :foo)
+      Process.exit(pid, :kill)
+      assert_receive {:DOWN, ^ref, _, _, _}
+      assert [] = PG.members(pg, :foo)
+    end
+
+    test "pg dies if other linked process dies", %{pg: pg} do
+      parent = self()
+      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+      ref = Process.monitor(pid)
+
+      temp =
+        spawn(fn ->
+          Process.link(pid)
+          send(parent, :continue)
+          Process.sleep(:infinity)
+        end)
+
+      assert_receive :continue
+
+      Process.exit(temp, :shutdown)
+      assert_receive {:DOWN, ^ref, _, _, _}
+    end
   end
 
-  test "processes are clened up when they die", %{pg: pg} do
-    {pid, ref} = spawn_monitor(Process, :sleep, [:infinity])
-    PG.track(pg, pid, :foo, :bar, :baz)
-    assert [_] = PG.list(pg, :foo)
-    Process.exit(pid, :kill)
-    assert_receive {:DOWN, ^ref, _, _, _}
-    assert [] = PG.list(pg, :foo)
+  describe "leave/2" do
+    test "removes entry", %{pg: pg} do
+      PG.join(pg, :foo, :bar, self(), :baz)
+
+      assert [_] = PG.members(pg, :foo)
+      assert PG.leave(pg, self()) == :ok
+      assert [] == PG.members(pg, :foo)
+    end
+
+    test "does not remove non members", %{pg: pg} do
+      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+      Process.link(pid)
+
+      assert PG.leave(pg, self()) == {:error, :not_member}
+      {:links, links} = Process.info(self(), :links)
+      assert pid in links
+    end
   end
 
-  test "pg dies if linked, untracked process terminates", %{pg: pg} do
-    parent = self()
-    [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
-    ref = Process.monitor(pid)
+  describe "leave/4" do
+    test "removes single entry", %{pg: pg} do
+      PG.join(pg, :foo, :bar, self(), :baz)
+      assert [_] = PG.members(pg, :foo)
+
+      assert PG.leave(pg, :foo, :bar, self()) == :ok
+      assert [] == PG.members(pg, :foo)
+    end
 
-    temp =
-      spawn(fn ->
-        Process.link(pid)
-        send(parent, :continue)
-        Process.sleep(:infinity)
-      end)
+    test "leaves other entries intact", %{pg: pg} do
+      PG.join(pg, :foo, :bar, self(), :baz)
+      PG.join(pg, :foo, :baar, self(), :baz)
+      assert [_, _] = PG.members(pg, :foo)
 
-    assert_receive :continue
+      assert PG.leave(pg, :foo, :bar, self()) == :ok
+      assert [{:baar, :baz}] == PG.members(pg, :foo)
+    end
 
-    Process.exit(temp, :shutdown)
-    assert_receive {:DOWN, ^ref, _, _, _}
+    test "does not remove non members", %{pg: pg} do
+      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+      Process.link(pid)
+
+      assert PG.leave(pg, :foo, :bar, self()) == {:error, :not_member}
+      {:links, links} = Process.info(self(), :links)
+      assert pid in links
+    end
   end
 
-  test "untrack/2", %{pg: pg} do
-    PG.track(pg, self(), :foo, :bar, :baz)
-    assert [_] = PG.list(pg, :foo)
-    PG.untrack(pg, self())
-    assert [] == PG.list(pg, :foo)
+  describe "update/5" do
+    test "executes the update if entry is present", %{pg: pg} do
+      parent = self()
+      PG.join(pg, :foo, :bar, self(), 1)
+      assert [{:bar, 1}] == PG.members(pg, :foo)
+
+      update = fn value ->
+        send(parent, value)
+        value + 1
+      end
+
+      assert PG.update(pg, :foo, :bar, self(), update) == :ok
+      assert_received 1
+      assert [{:bar, 2}] == PG.members(pg, :foo)
+    end
+
+    test "does not execute update if entry is absent", %{pg: pg} do
+      parent = self()
+      update = fn value -> Process.exit(parent, {:unexpected_update, value}) end
+      assert PG.update(pg, :foo, :bar, self(), update) == {:error, :not_member}
+    end
   end
 
-  test "untrack/4", %{pg: pg} do
-    PG.track(pg, self(), :foo, :bar, :baz)
-    assert [_] = PG.list(pg, :foo)
-    PG.untrack(pg, self(), :foo, :bar)
-    assert [] == PG.list(pg, :foo)
+  describe "replace/5" do
+    test "updates value if entry is present", %{pg: pg} do
+      PG.join(pg, :foo, :bar, self(), 1)
+      assert [{:bar, 1}] == PG.members(pg, :foo)
+      assert PG.replace(pg, :foo, :bar, self(), 2) == :ok
+      assert [{:bar, 2}] == PG.members(pg, :foo)
+    end
+
+    test "does not update value if entry is absent", %{pg: pg} do
+      assert PG.replace(pg, :foo, :bar, self(), 2) == {:error, :not_member}
+    end
   end
 end

From fb5b5304ea48659d0910bb183b561b95b24a5596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 16 May 2018 15:31:28 +0200
Subject: [PATCH 06/40] Fix start_supervised! only in 1.7

---
 test/firenest/pg_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
index 6cb1413..e60ef83 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/pg_test.exs
@@ -8,7 +8,7 @@ defmodule Firenest.PGTest do
   end
 
   setup %{test: test, topology: topology} do
-    start_supervised!({PG, name: test, topology: topology})
+    {:ok, _} = start_supervised({PG, name: test, topology: topology})
     {:ok, pg: test}
   end
 

From d5efd6e6a225f818ce3017f5580bf5d5872b115a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 16 May 2018 15:34:48 +0200
Subject: [PATCH 07/40] formatter

---
 lib/firenest/synced_server.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/firenest/synced_server.ex b/lib/firenest/synced_server.ex
index 7f3e5a8..9582978 100644
--- a/lib/firenest/synced_server.ex
+++ b/lib/firenest/synced_server.ex
@@ -410,7 +410,7 @@ defmodule Firenest.SyncedServer do
           :erlang.raise(:error, :undef, System.stacktrace())
         else
           {:registered_name, name} = Process.info(self(), :registered_name)
-          pattern = 'Undefined handle_info/2 in ~ts, process ~ts received unexpected message: ~p~n'
+          pattern = 'Undefined handle_info/2 in ~ts, process ~ts received message: ~p~n'
           :error_logger.warning_msg(pattern, [inspect(mod), inspect(name), hd(args)])
           {:noreply, int}
         end

From 6d9ff1a0f76898b4b781e0651c8eae8c859dd4f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 16 May 2018 15:53:28 +0200
Subject: [PATCH 08/40] Add specs to pg

---
 lib/firenest/pg.ex | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index 2cb40d7..9f54036 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -1,20 +1,26 @@
 defmodule Firenest.PG do
   alias Firenest.SyncedServer
 
-  @type pg() :: SyncedServer.server()
+  @type pg() :: atom()
+  @type group() :: term()
+  @type key() :: term()
+  @type value() :: term()
 
   defdelegate child_spec(opts), to: Firenest.PG.Supervisor
 
-  def join(pg, group, key, pid, meta) when node(pid) == node() do
+  @spec join(pg(), group(), key(), pid(), value()) :: :ok | {:error, :already_joined}
+  def join(pg, group, key, pid, value) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:join, group, key, pid, meta})
+    GenServer.call(server, {:join, group, key, pid, value})
   end
 
+  @spec leave(pg(), group(), key(), pid()) :: :ok | {:error, :not_member}
   def leave(pg, group, key, pid) when node(pid) == node() do
     server = partition_info!(pg, group)
     GenServer.call(server, {:leave, group, key, pid})
   end
 
+  @spec leave(pg(), pid()) :: :ok | {:error, :not_member}
   def leave(pg, pid) when node(pid) == node() do
     servers = partition_infos!(pg)
     replies = multicall(servers, {:leave, pid}, 5_000)
@@ -26,16 +32,19 @@ defmodule Firenest.PG do
     end
   end
 
+  @spec update(pg(), group(), key(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
   def update(pg, group, key, pid, update) when node(pid) == node() and is_function(update, 1) do
     server = partition_info!(pg, group)
     GenServer.call(server, {:update, group, key, pid, update})
   end
 
-  def replace(pg, group, key, pid, meta) when node(pid) == node() do
+  @spec replace(pg(), group(), key(), pid(), value()) :: :ok | {:error, :not_member}
+  def replace(pg, group, key, pid, value) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:replace, group, key, pid, meta})
+    GenServer.call(server, {:replace, group, key, pid, value})
   end
 
+  @spec members(pg(), group()) :: [{key(), value()}]
   def members(pg, group) do
     server = partition_info!(pg, group)
     GenServer.call(server, {:members, group}).()
@@ -145,7 +154,7 @@ defmodule Firenest.PG.Server do
   end
 
   @impl true
-  def handle_call({:join, group, key, pid, meta}, _from, state) do
+  def handle_call({:join, group, key, pid, value}, _from, state) do
     %{values: values, pids: pids} = state
     Process.link(pid)
     ets_key = {group, pid, key}
@@ -153,7 +162,7 @@ defmodule Firenest.PG.Server do
     if :ets.member(values, ets_key) do
       {:reply, {:error, :already_joined}, state}
     else
-      :ets.insert(values, {{group, pid, key}, meta})
+      :ets.insert(values, {{group, pid, key}, value})
       :ets.insert(pids, {group, pid, key})
       {:reply, :ok, state}
     end
@@ -192,12 +201,12 @@ defmodule Firenest.PG.Server do
     end
   end
 
-  def handle_call({:replace, group, key, pid, meta}, _from, state) do
+  def handle_call({:replace, group, key, pid, value}, _from, state) do
     %{values: values} = state
     ets_key = {group, pid, key}
 
     if :ets.member(values, ets_key) do
-      :ets.insert(values, {ets_key, meta})
+      :ets.insert(values, {ets_key, value})
       {:reply, :ok, state}
     else
       {:reply, {:error, :not_member}, state}

From e46a07f01ab445aeaafacdf28b60287c14352736 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 6 Jun 2018 12:28:06 +0200
Subject: [PATCH 09/40] Fix bug in SyncedServer where hello was not always sent

---
 lib/firenest/synced_server.ex        | 44 ++++++++++++++++------------
 test/firenest/synced_server_test.exs | 39 ++++++++++++++++++------
 test/support/eval_server.ex          |  1 +
 3 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/lib/firenest/synced_server.ex b/lib/firenest/synced_server.ex
index 9582978..7e4fc68 100644
--- a/lib/firenest/synced_server.ex
+++ b/lib/firenest/synced_server.ex
@@ -76,7 +76,7 @@ defmodule Firenest.SyncedServer do
 
   See `c:handle_info/2` for explanation of return values.
   """
-  @callback handle_replica(:up | :down, Topology.node_ref(), state()) ::
+  @callback handle_replica({:up, term()} | :down, Topology.node_ref(), state()) ::
               {:noreply, new_state}
               | {:noreply, new_state, timeout() | :hibernate}
               | {:stop, reason :: term(), new_state}
@@ -221,6 +221,7 @@ defmodule Firenest.SyncedServer do
   def init({mod, arg, topology, name}) do
     state = %{
       name: name,
+      node_ref: nil,
       topology: topology,
       mod: mod,
       int: nil,
@@ -230,7 +231,7 @@ defmodule Firenest.SyncedServer do
     }
 
     with {:ok, state} <- init_mod(mod, arg, state),
-         {:ok, state, node_ref} <- sync_named(topology, name, state) do
+         {:ok, %{node_ref: node_ref} = state} <- sync_named(topology, name, state) do
       Process.put(__MODULE__, {topology, name, node_ref})
       {:ok, state}
     end
@@ -252,25 +253,26 @@ defmodule Firenest.SyncedServer do
   end
 
   @impl true
-  def handle_info({:named_up, node_ref, name}, %{name: name} = state) do
-    %{awaiting_up: awaiting, replicas: replicas} = state
+  def handle_info({:named_up, remote_ref, name}, %{name: name} = state) do
+    %{awaiting_up: awaiting, replicas: replicas, topology: topology, node_ref: node_ref} = state
+    init_replicas(topology, name, [remote_ref], node_ref)
 
-    case delete_element(awaiting, node_ref) do
+    case delete_element(awaiting, remote_ref) do
       {:ok, awaiting} ->
-        result = apply_callback(state, :handle_replica, [:up, node_ref])
-        handle_common(result, %{state | awaiting_up: awaiting, replicas: [node_ref | replicas]})
+        result = apply_callback(state, :handle_replica, [:up, remote_ref])
+        handle_common(result, %{state | awaiting_up: awaiting, replicas: [remote_ref | replicas]})
 
       :error ->
-        {:noreply, %{state | awaiting_hello: [node_ref | awaiting]}}
+        {:noreply, %{state | awaiting_hello: [remote_ref | awaiting]}}
     end
   end
 
-  def handle_info({:named_down, node_ref, name}, %{name: name} = state) do
-    %{awaiting_hello: awaiting, replicas: replicas} = state
+  def handle_info({:named_down, remote_ref, name}, %{name: name} = state) do
+    %{awaiting_hello: awaiting, node_ref: node_ref, replicas: replicas} = state
 
-    case delete_element(replicas, node_ref) do
+    case delete_element(replicas, remote_ref) do
       {:ok, replicas} ->
-        result = apply_callback(state, :handle_replica, [:down, node_ref])
+        result = apply_callback(state, :handle_replica, [:down, remote_ref])
         handle_common(result, %{state | replicas: replicas})
 
       :error ->
@@ -278,17 +280,21 @@ defmodule Firenest.SyncedServer do
     end
   end
 
-  def handle_info({__MODULE__, :hello, node_ref}, state) do
-    %{awaiting_hello: awaiting, replicas: replicas} = state
+  def handle_info({__MODULE__, :hello, remote_ref}, state) do
+    %{awaiting_hello: awaiting, node_ref: node_ref, replicas: replicas} = state
 
-    case delete_element(awaiting, node_ref) do
+    case delete_element(awaiting, remote_ref) do
       {:ok, awaiting} ->
-        result = apply_callback(state, :handle_replica, [:up, node_ref])
+        result = apply_callback(state, :handle_replica, [:up, remote_ref])
 
-        handle_common(result, %{state | awaiting_hello: awaiting, replicas: [node_ref | replicas]})
+        handle_common(result, %{
+          state
+          | awaiting_hello: awaiting,
+            replicas: [remote_ref | replicas]
+        })
 
       :error ->
-        {:noreply, %{state | awaiting_up: [node_ref | awaiting]}}
+        {:noreply, %{state | awaiting_up: [remote_ref | awaiting]}}
     end
   end
 
@@ -387,7 +393,7 @@ defmodule Firenest.SyncedServer do
     case Topology.sync_named(topology, self()) do
       {:ok, replicas} ->
         init_replicas(topology, name, replicas, node_ref)
-        {:ok, %{state | awaiting_hello: replicas}, node_ref}
+        {:ok, %{state | awaiting_hello: replicas, node_ref: node_ref}}
 
       {:error, error} ->
         {:stop, {:sync_named, error}}
diff --git a/test/firenest/synced_server_test.exs b/test/firenest/synced_server_test.exs
index dbfd182..2e4007d 100644
--- a/test/firenest/synced_server_test.exs
+++ b/test/firenest/synced_server_test.exs
@@ -333,11 +333,35 @@ defmodule Firenest.SyncedServerTest do
 
     setup %{test: test, topology: topology} do
       {:ok, pid} = S.start_link(EvalServer, 1, name: test, topology: topology)
-      mfa = {S, :start_link, [EvalServer, 1, [name: test, topology: topology]]}
+      mfa = &{S, :start_link, [EvalServer, &1, [name: test, topology: topology]]}
       {:ok, mfa: mfa, pid: pid}
     end
 
     describe "handle_replica/3" do
+      test "both ends receive message", %{pid: pid, node: node} = config do
+        parent = self()
+
+        fun = fn status, replica ->
+          send(parent, {:replica, status, replica, 1})
+          {:noreply, 1}
+        end
+
+        remote_fun =
+          quote do
+            {:ok,
+             fn status, replica ->
+               send(unquote(parent), {:replica, status, replica, 2})
+               {:noreply, 2}
+             end}
+          end
+
+        send(pid, {:state, fun})
+        second = start_another(config, remote_fun)
+
+        assert_receive {:replica, :up, ^second, 1}
+        assert_receive {:replica, :up, ^node, 2}
+      end
+
       test "{:noreply, state}", %{pid: pid} = config do
         parent = self()
 
@@ -558,16 +582,13 @@ defmodule Firenest.SyncedServerTest do
     end
 
     defp start_another(config) do
-      %{test: test, mfa: mfa, nodes: [second | _]} = config
+      start_another(config, quote(do: {:ok, fn _, _ -> {:noreply, 1} end}))
+    end
 
-      cmd =
-        quote do
-          fun = fn _, _ -> {:noreply, 1} end
-          send(unquote(test), {:state, fun})
-        end
+    defp start_another(config, initial_state) do
+      %{mfa: mfa, nodes: [second | _]} = config
 
-      Firenest.Test.start_link([elem(second, 0)], mfa)
-      assert send_eval(config, second, cmd) == :ok
+      Firenest.Test.start_link([elem(second, 0)], mfa.({:eval, initial_state}))
       second
     end
 
diff --git a/test/support/eval_server.ex b/test/support/eval_server.ex
index 0b38bd8..f915286 100644
--- a/test/support/eval_server.ex
+++ b/test/support/eval_server.ex
@@ -2,6 +2,7 @@ defmodule Firenest.Test.EvalServer do
   use Firenest.SyncedServer
 
   def init(fun) when is_function(fun, 0), do: fun.()
+  def init({:eval, cmd}), do: elem(Code.eval_quoted(cmd), 0)
   def init(state), do: {:ok, state}
 
   def handle_call(:state, _, state), do: {:reply, state, state}

From ab0bea35a78d92379103c0f00a48995e7d82ef92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Thu, 7 Jun 2018 16:34:53 +0200
Subject: [PATCH 10/40] Add handshake_data callback to SyncedServer

---
 lib/firenest/synced_server.ex        | 54 ++++++++++++++++------------
 test/firenest/synced_server_test.exs | 14 ++++----
 test/support/eval_server.ex          |  3 ++
 3 files changed, 42 insertions(+), 29 deletions(-)

diff --git a/lib/firenest/synced_server.ex b/lib/firenest/synced_server.ex
index 7e4fc68..87737d4 100644
--- a/lib/firenest/synced_server.ex
+++ b/lib/firenest/synced_server.ex
@@ -65,6 +65,14 @@ defmodule Firenest.SyncedServer do
               | {:stop, reason :: term(), new_state}
             when new_state: state()
 
+  @doc """
+  Invoked before connecting to a remote synced server.
+
+  The returned value will be passed to the remote server in the
+  `c:handle_replica/3` callback.
+  """
+  @callback handshake_data(state()) :: term()
+
   @doc """
   Invoked when a status of a remote synced server changes.
 
@@ -226,12 +234,12 @@ defmodule Firenest.SyncedServer do
       mod: mod,
       int: nil,
       awaiting_hello: [],
-      awaiting_up: [],
+      awaiting_up: %{},
       replicas: []
     }
 
     with {:ok, state} <- init_mod(mod, arg, state),
-         {:ok, %{node_ref: node_ref} = state} <- sync_named(topology, name, state) do
+         {:ok, %{node_ref: node_ref} = state} <- sync_named(topology, state) do
       Process.put(__MODULE__, {topology, name, node_ref})
       {:ok, state}
     end
@@ -254,15 +262,17 @@ defmodule Firenest.SyncedServer do
 
   @impl true
   def handle_info({:named_up, remote_ref, name}, %{name: name} = state) do
-    %{awaiting_up: awaiting, replicas: replicas, topology: topology, node_ref: node_ref} = state
-    init_replicas(topology, name, [remote_ref], node_ref)
+    %{awaiting_up: awaiting, replicas: replicas} = state
+    init_replicas([remote_ref], state)
 
-    case delete_element(awaiting, remote_ref) do
-      {:ok, awaiting} ->
-        result = apply_callback(state, :handle_replica, [:up, remote_ref])
+    case awaiting do
+      %{^remote_ref => data} ->
+        result = apply_callback(state, :handle_replica, [{:up, data}, remote_ref])
+        awaiting = Map.delete(awaiting, remote_ref)
         handle_common(result, %{state | awaiting_up: awaiting, replicas: [remote_ref | replicas]})
 
-      :error ->
+      %{} ->
+        %{awaiting_hello: awaiting} = state
         {:noreply, %{state | awaiting_hello: [remote_ref | awaiting]}}
     end
   end
@@ -280,21 +290,19 @@ defmodule Firenest.SyncedServer do
     end
   end
 
-  def handle_info({__MODULE__, :hello, remote_ref}, state) do
-    %{awaiting_hello: awaiting, node_ref: node_ref, replicas: replicas} = state
+  def handle_info({__MODULE__, :hello, data, remote_ref}, state) do
+    %{awaiting_hello: awaiting, replicas: replicas} = state
 
     case delete_element(awaiting, remote_ref) do
       {:ok, awaiting} ->
-        result = apply_callback(state, :handle_replica, [:up, remote_ref])
+        result = apply_callback(state, :handle_replica, [{:up, data}, remote_ref])
 
-        handle_common(result, %{
-          state
-          | awaiting_hello: awaiting,
-            replicas: [remote_ref | replicas]
-        })
+        state = %{state | awaiting_hello: awaiting, replicas: [remote_ref | replicas]}
+        handle_common(result, state)
 
       :error ->
-        {:noreply, %{state | awaiting_up: [remote_ref | awaiting]}}
+        %{awaiting_up: awaiting} = state
+        {:noreply, %{state | awaiting_up: Map.put(awaiting, remote_ref, data)}}
     end
   end
 
@@ -387,21 +395,23 @@ defmodule Firenest.SyncedServer do
     end
   end
 
-  defp sync_named(topology, name, state) do
+  defp sync_named(topology, state) do
     node_ref = Topology.node(topology)
 
     case Topology.sync_named(topology, self()) do
       {:ok, replicas} ->
-        init_replicas(topology, name, replicas, node_ref)
-        {:ok, %{state | awaiting_hello: replicas, node_ref: node_ref}}
+        state = %{state | awaiting_hello: replicas, node_ref: node_ref}
+        init_replicas(replicas, state)
+        {:ok, state}
 
       {:error, error} ->
         {:stop, {:sync_named, error}}
     end
   end
 
-  defp init_replicas(topology, name, replicas, node_ref) do
-    Enum.each(replicas, &Topology.send(topology, &1, name, {__MODULE__, :hello, node_ref}))
+  defp init_replicas(replicas, %{topology: topology, name: name, node_ref: node_ref} = state) do
+    data = apply_callback(state, :handshake_data, [])
+    Enum.each(replicas, &Topology.send(topology, &1, name, {__MODULE__, :hello, data, node_ref}))
   end
 
   defp apply_callback(%{mod: mod, int: int}, fun, args) do
diff --git a/test/firenest/synced_server_test.exs b/test/firenest/synced_server_test.exs
index 2e4007d..5595fd7 100644
--- a/test/firenest/synced_server_test.exs
+++ b/test/firenest/synced_server_test.exs
@@ -358,8 +358,8 @@ defmodule Firenest.SyncedServerTest do
         send(pid, {:state, fun})
         second = start_another(config, remote_fun)
 
-        assert_receive {:replica, :up, ^second, 1}
-        assert_receive {:replica, :up, ^node, 2}
+        assert_receive {:replica, {:up, _}, ^second, 1}
+        assert_receive {:replica, {:up, _}, ^node, 2}
       end
 
       test "{:noreply, state}", %{pid: pid} = config do
@@ -373,7 +373,7 @@ defmodule Firenest.SyncedServerTest do
         send(pid, {:state, fun})
         second = start_another(config)
 
-        assert_receive {:replica, :up, ^second, 1}
+        assert_receive {:replica, {:up, _}, ^second, 1}
         assert S.call(pid, :state) == 1
       end
 
@@ -394,7 +394,7 @@ defmodule Firenest.SyncedServerTest do
         send(pid, {:state, fun})
         second = start_another(config)
 
-        assert_receive {:replica, :up, ^second, 1}
+        assert_receive {:replica, {:up, _}, ^second, 1}
         assert_receive {:timeout, 2}
         assert S.call(pid, :state) == 2
       end
@@ -411,7 +411,7 @@ defmodule Firenest.SyncedServerTest do
         second = start_another(config)
 
         assert_hibernate pid
-        assert_receive {:replica, :up, ^second, 1}
+        assert_receive {:replica, {:up, _}, ^second, 1}
         assert S.call(pid, :state) == 1
       end
 
@@ -432,7 +432,7 @@ defmodule Firenest.SyncedServerTest do
         send(pid, {:state, fun})
         second = start_another(config)
 
-        assert_receive {:replica, :up, ^second, 1}
+        assert_receive {:replica, {:up, _}, ^second, 1}
         assert_receive {:terminate, 1}
         assert_receive {:EXIT, ^pid, {:shutdown, _}}
       end
@@ -460,7 +460,7 @@ defmodule Firenest.SyncedServerTest do
           end
 
         assert send_eval(config, second, cmd) == :ok
-        assert_receive {:replica, :up, ^second, 1}
+        assert_receive {:replica, {:up, _}, ^second, 1}
         assert_receive {:replica, :down, ^second, 2}
         assert S.call(test, :state) == 2
       end
diff --git a/test/support/eval_server.ex b/test/support/eval_server.ex
index f915286..372e2f5 100644
--- a/test/support/eval_server.ex
+++ b/test/support/eval_server.ex
@@ -5,6 +5,9 @@ defmodule Firenest.Test.EvalServer do
   def init({:eval, cmd}), do: elem(Code.eval_quoted(cmd), 0)
   def init(state), do: {:ok, state}
 
+  def handshake_data(fun) when is_function(fun, 0), do: fun.()
+  def handshake_data(state), do: state
+
   def handle_call(:state, _, state), do: {:reply, state, state}
   def handle_call(fun, from, state), do: fun.(from, state)
 

From 84dbf5106374b3156c514de54e86824d99c7505a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 11 Jun 2018 12:55:38 +0200
Subject: [PATCH 11/40] First implementation of remote data transfer for PG

---
 lib/firenest/pg.ex        | 211 ++++++++++++++++++++++++++++++++++----
 test/firenest/pg_test.exs |  43 +++++++-
 2 files changed, 233 insertions(+), 21 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index 9f54036..aeb2d25 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -6,18 +6,29 @@ defmodule Firenest.PG do
   @type key() :: term()
   @type value() :: term()
 
+  @doc """
+
+  ## Options
+
+    * `:name` - name for the process, required;
+    * `:topology` - name of the supporting topology, required;
+    * `:partitions` - number of partitions, defaults to 1;
+    * `:broadcast_timeout` - delay of broadcasting local events to other
+      nodes, defaults to 50 ms;
+
+  """
   defdelegate child_spec(opts), to: Firenest.PG.Supervisor
 
   @spec join(pg(), group(), key(), pid(), value()) :: :ok | {:error, :already_joined}
   def join(pg, group, key, pid, value) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:join, group, key, pid, value})
+    SyncedServer.call(server, {:join, group, key, pid, value})
   end
 
   @spec leave(pg(), group(), key(), pid()) :: :ok | {:error, :not_member}
   def leave(pg, group, key, pid) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:leave, group, key, pid})
+    SyncedServer.call(server, {:leave, group, key, pid})
   end
 
   @spec leave(pg(), pid()) :: :ok | {:error, :not_member}
@@ -35,19 +46,19 @@ defmodule Firenest.PG do
   @spec update(pg(), group(), key(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
   def update(pg, group, key, pid, update) when node(pid) == node() and is_function(update, 1) do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:update, group, key, pid, update})
+    SyncedServer.call(server, {:update, group, key, pid, update})
   end
 
   @spec replace(pg(), group(), key(), pid(), value()) :: :ok | {:error, :not_member}
   def replace(pg, group, key, pid, value) when node(pid) == node() do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:replace, group, key, pid, value})
+    SyncedServer.call(server, {:replace, group, key, pid, value})
   end
 
   @spec members(pg(), group()) :: [{key(), value()}]
   def members(pg, group) do
     server = partition_info!(pg, group)
-    GenServer.call(server, {:members, group}).()
+    SyncedServer.call(server, {:members, group}).()
   end
 
   # TODO
@@ -146,13 +157,27 @@ defmodule Firenest.PG.Server do
   end
 
   @impl true
-  def init({name, _opts}) do
+  def init({name, opts}) do
     Process.flag(:trap_exit, true)
     values = :ets.new(name, [:named_table, :protected, :ordered_set])
     pids = :ets.new(__MODULE__.Pids, [:duplicate_bag, keypos: 2])
-    {:ok, %{values: values, pids: pids}}
+    broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
+
+    {:ok,
+     %{
+       values: values,
+       pids: pids,
+       broadcast_timer: nil,
+       broadcast_timeout: broadcast_timeout,
+       clock: 0,
+       remote_clocks: %{},
+       pending_events: []
+     }}
   end
 
+  @impl true
+  def handshake_data(%{clock: clock}), do: clock
+
   @impl true
   def handle_call({:join, group, key, pid, value}, _from, state) do
     %{values: values, pids: pids} = state
@@ -164,6 +189,7 @@ defmodule Firenest.PG.Server do
     else
       :ets.insert(values, {{group, pid, key}, value})
       :ets.insert(pids, {group, pid, key})
+      state = schedule_broadcast_events(state, [{:join, group, key, pid, value}])
       {:reply, :ok, state}
     end
   end
@@ -183,6 +209,7 @@ defmodule Firenest.PG.Server do
         end
 
         :ets.delete(values, key)
+        state = schedule_broadcast_events(state, [{:leave, group, key, pid}])
         {:reply, :ok, state}
     end
   end
@@ -193,7 +220,9 @@ defmodule Firenest.PG.Server do
 
     case ets_fetch_element(values, ets_key, 2) do
       {:ok, value} ->
-        :ets.insert(values, {ets_key, update.(value)})
+        new_value = update.(value)
+        :ets.insert(values, {ets_key, new_value})
+        state = schedule_broadcast_events(state, [{:replace, group, key, pid, new_value}])
         {:reply, :ok, state}
 
       :error ->
@@ -207,6 +236,7 @@ defmodule Firenest.PG.Server do
 
     if :ets.member(values, ets_key) do
       :ets.insert(values, {ets_key, value})
+      state = schedule_broadcast_events(state, [{:replace, group, key, pid, value}])
       {:reply, :ok, state}
     else
       {:reply, {:error, :not_member}, state}
@@ -216,11 +246,14 @@ defmodule Firenest.PG.Server do
   def handle_call({:leave, pid}, _from, state) do
     %{values: values, pids: pids} = state
 
-    if untrack_pid(pids, values, pid) do
-      Process.unlink(pid)
-      {:reply, :ok, state}
-    else
-      {:reply, {:error, :not_member}, state}
+    case untrack_pid(pids, values, pid) do
+      {:ok, leaves} ->
+        unlink_flush(pid)
+        state = schedule_broadcast_events(state, leaves)
+        {:reply, :ok, state}
+
+      :error ->
+        {:reply, {:error, :not_member}, state}
     end
   end
 
@@ -228,8 +261,9 @@ defmodule Firenest.PG.Server do
     %{values: values} = state
 
     read = fn ->
-      ms = [{{{group, :_, :"$1"}, :"$2"}, [], [{{:"$1", :"$2"}}]}]
-      :ets.select(values, ms)
+      local = {{{group, :_, :"$1"}, :"$2"}, [], [{{:"$1", :"$2"}}]}
+      remote = {{{group, :_, :"$1"}, :_, :"$2"}, [], [{{:"$1", :"$2"}}]}
+      :ets.select(values, [local, remote])
     end
 
     {:reply, read, state}
@@ -239,22 +273,93 @@ defmodule Firenest.PG.Server do
   def handle_info({:EXIT, pid, reason}, state) do
     %{values: values, pids: pids} = state
 
-    if untrack_pid(pids, values, pid) do
-      {:noreply, state}
+    case untrack_pid(pids, values, pid) do
+      {:ok, leaves} ->
+        state = schedule_broadcast_events(state, leaves)
+        {:noreply, state}
+
+      :error ->
+        {:stop, reason, state}
+    end
+  end
+
+  def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
+    %{pending_events: events, clock: clock} = state
+    clock = clock + 1
+    SyncedServer.remote_broadcast({:events, clock, events})
+    {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
+  end
+
+  @impl true
+  def handle_remote({:catch_up_req, clock}, from, state) do
+    {mode, data} = catch_up_reply(state, clock)
+    SyncedServer.remote_send(from, {:catch_up, mode, data})
+    {:noreply, state}
+  end
+
+  def handle_remote({:catch_up, :state_transfer, {clock, transfer}}, from, state) do
+    state = handle_state_transfer(state, from, clock, transfer)
+    {:noreply, state}
+  end
+
+  def handle_remote({:events, remote_clock, events}, from, state) do
+    %{remote_clocks: remote_clocks} = state
+    local_clock = Map.fetch!(remote_clocks, from)
+
+    if remote_clock == local_clock + 1 do
+      remote_clocks = %{remote_clocks | from => remote_clock}
+      state = handle_events(state, from, events)
+      {:noreply, %{state | remote_clocks: remote_clocks}}
     else
-      {:stop, reason, state}
+      {:noreply, request_catch_up(state, from, local_clock)}
+    end
+  end
+
+  @impl true
+  def handle_replica({:up, remote_clock}, remote_ref, state) do
+    %{remote_clocks: remote_clocks} = state
+
+    case remote_clocks do
+      %{^remote_ref => old_clock} when remote_clock > old_clock ->
+        # Reconnection, try to catch up
+        {:noreply, request_catch_up(state, remote_ref, old_clock)}
+
+      %{^remote_ref => old_clock} ->
+        # Reconnection, no remote state change, skip catch up
+        # Assert for sanity
+        true = old_clock == remote_clock
+        {:noreply, state}
+
+      %{} when remote_clock == 0 ->
+        # New node, no state, don't catch up
+        state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
+        {:noreply, state}
+
+      %{} ->
+        # New node, catch up
+        state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
+        {:noreply, request_catch_up(state, remote_ref, 0)}
     end
   end
 
+  # TODO: remove data from that node
+  def handle_replica(:down, remote_ref, state) do
+    %{values: values, remote_clocks: remote_clocks} = state
+    delete_ms = [{{:_, remote_ref, :_}, [], [true]}]
+    :ets.select_delete(values, delete_ms)
+    {:noreply, %{state | remote_clocks: Map.delete(remote_clocks, remote_ref)}}
+  end
+
   defp untrack_pid(pids, values, pid) do
     case :ets.take(pids, pid) do
       [] ->
-        false
+        :error
 
       list ->
         ms = for key <- list, do: {{key, :_}, [], [true]}
         :ets.select_delete(values, ms)
-        true
+        leaves = for {group, pid, key} <- list, do: {:leave, group, key, pid}
+        {:ok, leaves}
     end
   end
 
@@ -263,4 +368,70 @@ defmodule Firenest.PG.Server do
   catch
     :error, :badarg -> :error
   end
+
+  defp unlink_flush(pid) do
+    Process.unlink(pid)
+
+    receive do
+      {:EXIT, ^pid, _} -> :ok
+    after
+      0 -> :ok
+    end
+  end
+
+  defp schedule_broadcast_events(%{broadcast_timer: nil} = state, new_events) do
+    %{broadcast_timeout: timeout, pending_events: events} = state
+    timer = :erlang.start_timer(timeout, self(), :broadcast)
+    %{state | broadcast_timer: timer, pending_events: new_events ++ events}
+  end
+
+  defp schedule_broadcast_events(%{} = state, new_events) do
+    %{pending_events: events} = state
+    %{state | pending_events: new_events ++ events}
+  end
+
+  defp request_catch_up(state, remote_ref, clock) do
+    SyncedServer.remote_send(remote_ref, {:catch_up_req, clock})
+    state
+  end
+
+  defp handle_events(%{values: values} = state, from, events) do
+    {joins, leaves} =
+      Enum.reduce(events, {[], []}, fn
+        {:leave, group, key, pid}, {joins, leaves} ->
+          leave = {{{group, pid, key}, from, :_}, [], [true]}
+          {joins, [leave | leaves]}
+
+        {:replace, group, key, pid, value}, {joins, leaves} ->
+          join = {{group, pid, key}, from, value}
+          {[join | joins], leaves}
+
+        {:join, group, key, pid, value}, {joins, leaves} ->
+          join = {{group, pid, key}, from, value}
+          {[join | joins], leaves}
+      end)
+
+    :ets.insert(values, joins)
+    :ets.select_delete(values, leaves)
+    state
+  end
+
+  # TODO: detect leaves
+  # Is there a better way than to clean up and re-insert?
+  # This can be problematic for dirty reads!
+  defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
+    %{remote_clocks: remote_clocks} = state
+    delete_ms = [{{:_, from, :_}, [], [true]}]
+    inserts = for {key, value} <- transfer, do: {key, from, value}
+    :ets.select_delete(values, delete_ms)
+    :ets.insert(values, inserts)
+    state
+    %{state | remote_clocks: %{remote_clocks | from => clock}}
+  end
+
+  # TODO: handle catch-up with events
+  defp catch_up_reply(%{values: values}, _clock) do
+    local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
+    {:state_transfer, :ets.select(values, local_ms)}
+  end
 end
diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
index e60ef83..90d7409 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/pg_test.exs
@@ -1,14 +1,17 @@
 defmodule Firenest.PGTest do
   use ExUnit.Case, async: true
 
+  alias Firenest.Topology, as: T
   alias Firenest.PG
 
+  import Firenest.TestHelpers
+
   setup_all do
     {:ok, topology: Firenest.Test, evaluator: Firenest.Test.Evaluator}
   end
 
   setup %{test: test, topology: topology} do
-    {:ok, _} = start_supervised({PG, name: test, topology: topology})
+    assert {:ok, _} = start_supervised({PG, name: test, topology: topology})
     {:ok, pg: test}
   end
 
@@ -133,4 +136,42 @@ defmodule Firenest.PGTest do
       assert PG.replace(pg, :foo, :bar, self(), 2) == {:error, :not_member}
     end
   end
+
+  defmodule Distributed do
+    use ExUnit.Case, async: true
+
+    setup_all do
+      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
+      nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
+      topology = Firenest.Test
+      pg = Firenest.Test.PG
+      %{start: start} = PG.child_spec(name: pg, topology: topology)
+      Firenest.Test.start_link(nodes, start)
+      nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
+
+      {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, pg: pg}
+    end
+
+    setup %{test: test} do
+      {:ok, group: test}
+    end
+
+    test "remote join is propagated", config do
+      %{topology: topology, evaluator: evaluator, pg: pg, group: group, nodes: [second]} = config
+
+      cmd =
+        quote do
+          spawn(fn ->
+            :ok = PG.join(unquote(pg), unquote(group), :bar, self(), :baz)
+            :timer.sleep(:infinity)
+          end)
+        end
+
+      T.send(topology, second, evaluator, {:eval_quoted, cmd})
+
+      :timer.sleep(1_000)
+      assert PG.members(pg, group) == [{:bar, :baz}]
+      # wait_until(fn -> PG.members(pg, group) == [{:bar, :baz}] end)
+    end
+  end
 end

From c474be80b836e75f573deeb00162b1f9939b0c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 11 Jun 2018 19:05:06 +0200
Subject: [PATCH 12/40] Remove key from PG

---
 lib/firenest/pg.ex        | 85 +++++++++++++++++++--------------------
 test/firenest/pg_test.exs | 53 ++++++++++++------------
 2 files changed, 67 insertions(+), 71 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index aeb2d25..ff91c46 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -3,7 +3,6 @@ defmodule Firenest.PG do
 
   @type pg() :: atom()
   @type group() :: term()
-  @type key() :: term()
   @type value() :: term()
 
   @doc """
@@ -19,16 +18,16 @@ defmodule Firenest.PG do
   """
   defdelegate child_spec(opts), to: Firenest.PG.Supervisor
 
-  @spec join(pg(), group(), key(), pid(), value()) :: :ok | {:error, :already_joined}
-  def join(pg, group, key, pid, value) when node(pid) == node() do
+  @spec join(pg(), group(), pid(), value()) :: :ok | {:error, :already_joined}
+  def join(pg, group, pid, value) when node(pid) == node() do
     server = partition_info!(pg, group)
-    SyncedServer.call(server, {:join, group, key, pid, value})
+    SyncedServer.call(server, {:join, group, pid, value})
   end
 
-  @spec leave(pg(), group(), key(), pid()) :: :ok | {:error, :not_member}
-  def leave(pg, group, key, pid) when node(pid) == node() do
+  @spec leave(pg(), group(), pid()) :: :ok | {:error, :not_member}
+  def leave(pg, group, pid) when node(pid) == node() do
     server = partition_info!(pg, group)
-    SyncedServer.call(server, {:leave, group, key, pid})
+    SyncedServer.call(server, {:leave, group, pid})
   end
 
   @spec leave(pg(), pid()) :: :ok | {:error, :not_member}
@@ -43,19 +42,19 @@ defmodule Firenest.PG do
     end
   end
 
-  @spec update(pg(), group(), key(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
-  def update(pg, group, key, pid, update) when node(pid) == node() and is_function(update, 1) do
+  @spec update(pg(), group(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
+  def update(pg, group, pid, update) when node(pid) == node() and is_function(update, 1) do
     server = partition_info!(pg, group)
-    SyncedServer.call(server, {:update, group, key, pid, update})
+    SyncedServer.call(server, {:update, group, pid, update})
   end
 
-  @spec replace(pg(), group(), key(), pid(), value()) :: :ok | {:error, :not_member}
-  def replace(pg, group, key, pid, value) when node(pid) == node() do
+  @spec replace(pg(), group(), pid(), value()) :: :ok | {:error, :not_member}
+  def replace(pg, group, pid, value) when node(pid) == node() do
     server = partition_info!(pg, group)
-    SyncedServer.call(server, {:replace, group, key, pid, value})
+    SyncedServer.call(server, {:replace, group, pid, value})
   end
 
-  @spec members(pg(), group()) :: [{key(), value()}]
+  @spec members(pg(), group()) :: [value()]
   def members(pg, group) do
     server = partition_info!(pg, group)
     SyncedServer.call(server, {:members, group}).()
@@ -179,24 +178,24 @@ defmodule Firenest.PG.Server do
   def handshake_data(%{clock: clock}), do: clock
 
   @impl true
-  def handle_call({:join, group, key, pid, value}, _from, state) do
+  def handle_call({:join, group, pid, value}, _from, state) do
     %{values: values, pids: pids} = state
     Process.link(pid)
-    ets_key = {group, pid, key}
+    key = {group, pid}
 
-    if :ets.member(values, ets_key) do
+    if :ets.member(values, key) do
       {:reply, {:error, :already_joined}, state}
     else
-      :ets.insert(values, {{group, pid, key}, value})
-      :ets.insert(pids, {group, pid, key})
-      state = schedule_broadcast_events(state, [{:join, group, key, pid, value}])
+      :ets.insert(values, {key, value})
+      :ets.insert(pids, key)
+      state = schedule_broadcast_events(state, [{:join, group, pid, value}])
       {:reply, :ok, state}
     end
   end
 
-  def handle_call({:leave, group, key, pid}, _from, state) do
+  def handle_call({:leave, group, pid}, _from, state) do
     %{values: values, pids: pids} = state
-    key = {group, pid, key}
+    key = {group, pid}
     ms = [{key, [], [true]}]
 
     case :ets.select_delete(pids, ms) do
@@ -209,20 +208,20 @@ defmodule Firenest.PG.Server do
         end
 
         :ets.delete(values, key)
-        state = schedule_broadcast_events(state, [{:leave, group, key, pid}])
+        state = schedule_broadcast_events(state, [{:leave, group, pid}])
         {:reply, :ok, state}
     end
   end
 
-  def handle_call({:update, group, key, pid, update}, _from, state) do
+  def handle_call({:update, group, pid, update}, _from, state) do
     %{values: values} = state
-    ets_key = {group, pid, key}
+    key = {group, pid}
 
-    case ets_fetch_element(values, ets_key, 2) do
+    case ets_fetch_element(values, key, 2) do
       {:ok, value} ->
         new_value = update.(value)
-        :ets.insert(values, {ets_key, new_value})
-        state = schedule_broadcast_events(state, [{:replace, group, key, pid, new_value}])
+        :ets.insert(values, {key, new_value})
+        state = schedule_broadcast_events(state, [{:replace, group, pid, new_value}])
         {:reply, :ok, state}
 
       :error ->
@@ -230,13 +229,13 @@ defmodule Firenest.PG.Server do
     end
   end
 
-  def handle_call({:replace, group, key, pid, value}, _from, state) do
+  def handle_call({:replace, group, pid, value}, _from, state) do
     %{values: values} = state
-    ets_key = {group, pid, key}
+    key = {group, pid}
 
-    if :ets.member(values, ets_key) do
-      :ets.insert(values, {ets_key, value})
-      state = schedule_broadcast_events(state, [{:replace, group, key, pid, value}])
+    if :ets.member(values, key) do
+      :ets.insert(values, {key, value})
+      state = schedule_broadcast_events(state, [{:replace, group, pid, value}])
       {:reply, :ok, state}
     else
       {:reply, {:error, :not_member}, state}
@@ -261,8 +260,8 @@ defmodule Firenest.PG.Server do
     %{values: values} = state
 
     read = fn ->
-      local = {{{group, :_, :"$1"}, :"$2"}, [], [{{:"$1", :"$2"}}]}
-      remote = {{{group, :_, :"$1"}, :_, :"$2"}, [], [{{:"$1", :"$2"}}]}
+      local = {{{group, :_}, :"$1"}, [], [:"$1"]}
+      remote = {{{group, :_}, :_, :"$1"}, [], [:"$1"]}
       :ets.select(values, [local, remote])
     end
 
@@ -342,7 +341,6 @@ defmodule Firenest.PG.Server do
     end
   end
 
-  # TODO: remove data from that node
   def handle_replica(:down, remote_ref, state) do
     %{values: values, remote_clocks: remote_clocks} = state
     delete_ms = [{{:_, remote_ref, :_}, [], [true]}]
@@ -358,7 +356,7 @@ defmodule Firenest.PG.Server do
       list ->
         ms = for key <- list, do: {{key, :_}, [], [true]}
         :ets.select_delete(values, ms)
-        leaves = for {group, pid, key} <- list, do: {:leave, group, key, pid}
+        leaves = for {group, pid} <- list, do: {:leave, group, pid}
         {:ok, leaves}
     end
   end
@@ -398,16 +396,16 @@ defmodule Firenest.PG.Server do
   defp handle_events(%{values: values} = state, from, events) do
     {joins, leaves} =
       Enum.reduce(events, {[], []}, fn
-        {:leave, group, key, pid}, {joins, leaves} ->
-          leave = {{{group, pid, key}, from, :_}, [], [true]}
+        {:leave, group, pid}, {joins, leaves} ->
+          leave = {{{group, pid}, from, :_}, [], [true]}
           {joins, [leave | leaves]}
 
-        {:replace, group, key, pid, value}, {joins, leaves} ->
-          join = {{group, pid, key}, from, value}
+        {:replace, group, pid, value}, {joins, leaves} ->
+          join = {{group, pid}, from, value}
           {[join | joins], leaves}
 
-        {:join, group, key, pid, value}, {joins, leaves} ->
-          join = {{group, pid, key}, from, value}
+        {:join, group, pid, value}, {joins, leaves} ->
+          join = {{group, pid}, from, value}
           {[join | joins], leaves}
       end)
 
@@ -425,7 +423,6 @@ defmodule Firenest.PG.Server do
     inserts = for {key, value} <- transfer, do: {key, from, value}
     :ets.select_delete(values, delete_ms)
     :ets.insert(values, inserts)
-    state
     %{state | remote_clocks: %{remote_clocks | from => clock}}
   end
 
diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
index 90d7409..508302e 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/pg_test.exs
@@ -17,18 +17,18 @@ defmodule Firenest.PGTest do
 
   describe "join/5" do
     test "adds process", %{pg: pg} do
-      assert PG.join(pg, :foo, :bar, self(), :baz) == :ok
-      assert [{:bar, :baz}] == PG.members(pg, :foo)
+      assert PG.join(pg, :foo, self(), :baz) == :ok
+      assert [:baz] == PG.members(pg, :foo)
     end
 
     test "rejects double joins", %{pg: pg} do
-      assert PG.join(pg, :foo, :bar, self(), :baz) == :ok
-      assert PG.join(pg, :foo, :bar, self(), :baz) == {:error, :already_joined}
+      assert PG.join(pg, :foo, self(), :baz) == :ok
+      assert PG.join(pg, :foo, self(), :baz) == {:error, :already_joined}
     end
 
     test "cleans up entries after process dies", %{pg: pg} do
       {pid, ref} = spawn_monitor(Process, :sleep, [:infinity])
-      PG.join(pg, :foo, :bar, pid, :baz)
+      PG.join(pg, :foo, pid, :baz)
       assert [_] = PG.members(pg, :foo)
       Process.exit(pid, :kill)
       assert_receive {:DOWN, ^ref, _, _, _}
@@ -56,7 +56,7 @@ defmodule Firenest.PGTest do
 
   describe "leave/2" do
     test "removes entry", %{pg: pg} do
-      PG.join(pg, :foo, :bar, self(), :baz)
+      PG.join(pg, :foo, self(), :baz)
 
       assert [_] = PG.members(pg, :foo)
       assert PG.leave(pg, self()) == :ok
@@ -75,27 +75,28 @@ defmodule Firenest.PGTest do
 
   describe "leave/4" do
     test "removes single entry", %{pg: pg} do
-      PG.join(pg, :foo, :bar, self(), :baz)
+      PG.join(pg, :foo, self(), :baz)
       assert [_] = PG.members(pg, :foo)
 
-      assert PG.leave(pg, :foo, :bar, self()) == :ok
+      assert PG.leave(pg, :foo, self()) == :ok
       assert [] == PG.members(pg, :foo)
     end
 
     test "leaves other entries intact", %{pg: pg} do
-      PG.join(pg, :foo, :bar, self(), :baz)
-      PG.join(pg, :foo, :baar, self(), :baz)
+      pid = spawn_link(fn -> Process.sleep(:infinity) end)
+      PG.join(pg, :foo, self(), :baz)
+      PG.join(pg, :foo, pid, :baaz)
       assert [_, _] = PG.members(pg, :foo)
 
-      assert PG.leave(pg, :foo, :bar, self()) == :ok
-      assert [{:baar, :baz}] == PG.members(pg, :foo)
+      assert PG.leave(pg, :foo, self()) == :ok
+      assert [:baaz] == PG.members(pg, :foo)
     end
 
     test "does not remove non members", %{pg: pg} do
       [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
       Process.link(pid)
 
-      assert PG.leave(pg, :foo, :bar, self()) == {:error, :not_member}
+      assert PG.leave(pg, :foo, self()) == {:error, :not_member}
       {:links, links} = Process.info(self(), :links)
       assert pid in links
     end
@@ -104,36 +105,36 @@ defmodule Firenest.PGTest do
   describe "update/5" do
     test "executes the update if entry is present", %{pg: pg} do
       parent = self()
-      PG.join(pg, :foo, :bar, self(), 1)
-      assert [{:bar, 1}] == PG.members(pg, :foo)
+      PG.join(pg, :foo, self(), 1)
+      assert [1] == PG.members(pg, :foo)
 
       update = fn value ->
         send(parent, value)
         value + 1
       end
 
-      assert PG.update(pg, :foo, :bar, self(), update) == :ok
+      assert PG.update(pg, :foo, self(), update) == :ok
       assert_received 1
-      assert [{:bar, 2}] == PG.members(pg, :foo)
+      assert [2] == PG.members(pg, :foo)
     end
 
     test "does not execute update if entry is absent", %{pg: pg} do
       parent = self()
       update = fn value -> Process.exit(parent, {:unexpected_update, value}) end
-      assert PG.update(pg, :foo, :bar, self(), update) == {:error, :not_member}
+      assert PG.update(pg, :foo, self(), update) == {:error, :not_member}
     end
   end
 
   describe "replace/5" do
     test "updates value if entry is present", %{pg: pg} do
-      PG.join(pg, :foo, :bar, self(), 1)
-      assert [{:bar, 1}] == PG.members(pg, :foo)
-      assert PG.replace(pg, :foo, :bar, self(), 2) == :ok
-      assert [{:bar, 2}] == PG.members(pg, :foo)
+      PG.join(pg, :foo, self(), 1)
+      assert [1] == PG.members(pg, :foo)
+      assert PG.replace(pg, :foo, self(), 2) == :ok
+      assert [2] == PG.members(pg, :foo)
     end
 
     test "does not update value if entry is absent", %{pg: pg} do
-      assert PG.replace(pg, :foo, :bar, self(), 2) == {:error, :not_member}
+      assert PG.replace(pg, :foo, self(), 2) == {:error, :not_member}
     end
   end
 
@@ -162,16 +163,14 @@ defmodule Firenest.PGTest do
       cmd =
         quote do
           spawn(fn ->
-            :ok = PG.join(unquote(pg), unquote(group), :bar, self(), :baz)
+            :ok = PG.join(unquote(pg), unquote(group), self(), :baz)
             :timer.sleep(:infinity)
           end)
         end
 
       T.send(topology, second, evaluator, {:eval_quoted, cmd})
 
-      :timer.sleep(1_000)
-      assert PG.members(pg, group) == [{:bar, :baz}]
-      # wait_until(fn -> PG.members(pg, group) == [{:bar, :baz}] end)
+      wait_until(fn -> PG.members(pg, group) == [:baz] end)
     end
   end
 end

From d9e2b613fff688b660d6dfaba94008ca793bb169 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 12 Jun 2018 20:45:30 +0200
Subject: [PATCH 13/40] Add up/down/up test to PG

---
 lib/firenest/pg.ex           | 12 ++++--
 test/firenest/pg_test.exs    | 80 ++++++++++++++++++++++++++++++------
 test/shared/test.ex          | 15 +++----
 test/support/test_helpers.ex |  2 +-
 4 files changed, 84 insertions(+), 25 deletions(-)

diff --git a/lib/firenest/pg.ex b/lib/firenest/pg.ex
index ff91c46..f247ba3 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/pg.ex
@@ -164,7 +164,7 @@ defmodule Firenest.PG.Server do
 
     {:ok,
      %{
-       values: values,
+       values: ets_whereis(values),
        pids: pids,
        broadcast_timer: nil,
        broadcast_timeout: broadcast_timeout,
@@ -427,8 +427,14 @@ defmodule Firenest.PG.Server do
   end
 
   # TODO: handle catch-up with events
-  defp catch_up_reply(%{values: values}, _clock) do
+  defp catch_up_reply(%{values: values}, clock) do
     local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
-    {:state_transfer, :ets.select(values, local_ms)}
+    {:state_transfer, {clock, :ets.select(values, local_ms)}}
+  end
+
+  if function_exported?(:ets, :whereis, 1) do
+    defp ets_whereis(table), do: :ets.whereis(table)
+  else
+    defp ets_whereis(table), do: table
   end
 end
diff --git a/test/firenest/pg_test.exs b/test/firenest/pg_test.exs
index 508302e..a9def0f 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/pg_test.exs
@@ -139,11 +139,12 @@ defmodule Firenest.PGTest do
   end
 
   defmodule Distributed do
-    use ExUnit.Case, async: true
+    # We modify test topology, it can't be async
+    use ExUnit.Case
 
     setup_all do
       wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
-      nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
+      nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
       topology = Firenest.Test
       pg = Firenest.Test.PG
       %{start: start} = PG.child_spec(name: pg, topology: topology)
@@ -158,19 +159,74 @@ defmodule Firenest.PGTest do
     end
 
     test "remote join is propagated", config do
-      %{topology: topology, evaluator: evaluator, pg: pg, group: group, nodes: [second]} = config
+      %{pg: pg, group: group, nodes: [second | _]} = config
 
-      cmd =
-        quote do
-          spawn(fn ->
-            :ok = PG.join(unquote(pg), unquote(group), self(), :baz)
-            :timer.sleep(:infinity)
-          end)
-        end
-
-      T.send(topology, second, evaluator, {:eval_quoted, cmd})
+      quote do
+        spawn(fn ->
+          :ok = PG.join(unquote(pg), unquote(group), self(), :baz)
+          :timer.sleep(:infinity)
+        end)
+      end
+      |> eval_on_node(second, config)
 
       wait_until(fn -> PG.members(pg, group) == [:baz] end)
     end
+
+    test "propages changes when nodes were disconnected", config do
+      %{topology: topology, pg: pg, test: test, nodes: [second, third]} = config
+      Process.register(self(), test)
+
+      quote do
+        spawn(fn ->
+          Process.register(self(), unquote(test))
+          :ok = PG.join(unquote(pg), unquote(test), self(), :baz)
+          Process.sleep(:infinity)
+        end)
+      end
+      |> eval_on_node(second, config)
+
+      quote(do: PG.members(unquote(pg), unquote(test)) == [:baz])
+      |> await_on_node(third, config)
+
+      quote do
+        T.disconnect(unquote(topology), elem(unquote(third), 0))
+        pid = Process.whereis(unquote(test))
+        :ok = PG.leave(unquote(pg), unquote(test), pid)
+
+        spawn(fn ->
+          :ok = PG.join(unquote(pg), unquote(test), self(), :bar)
+          Process.sleep(:infinity)
+        end)
+      end
+      |> eval_on_node(second, config)
+
+      quote(do: PG.members(unquote(pg), unquote(test)) == [])
+      |> await_on_node(third, config)
+
+      quote(do: T.connect(unquote(topology), elem(unquote(third), 0)))
+      |> eval_on_node(second, config)
+
+      quote(do: PG.members(unquote(pg), unquote(test)) == [:bar])
+      |> await_on_node(third, config)
+    end
+
+    defp eval_on_node(quoted, node, config) do
+      %{topology: topology, evaluator: evaluator} = config
+
+      T.send(topology, node, evaluator, {:eval_quoted, quoted})
+    end
+
+    defp await_on_node(quoted, node, config) do
+      %{topology: topology} = config
+      {:registered_name, name} = Process.info(self(), :registered_name)
+
+      quote do
+        wait_until(fn -> unquote(quoted) end)
+        T.broadcast(unquote(topology), unquote(name), :continue)
+      end
+      |> eval_on_node(node, config)
+
+      assert_receive :continue
+    end
   end
 end
diff --git a/test/shared/test.ex b/test/shared/test.ex
index b53aa2b..4dfd3eb 100644
--- a/test/shared/test.ex
+++ b/test/shared/test.ex
@@ -25,6 +25,11 @@ defmodule Firenest.Test do
     def handle_info({:eval_quoted, quoted}, state) do
       Code.eval_quoted(quoted)
       {:noreply, state}
+    catch
+      kind, reason ->
+        exception = Exception.format(kind, reason, System.stacktrace())
+        Logger.error("Eval failed on node #{inspect node()}\n#{exception}")
+        {:noreply, state}
     end
 
     def handle_info(_, state) do
@@ -84,21 +89,13 @@ defmodule Firenest.Test do
 
     spawn_link(fn ->
       Process.register(self(), __MODULE__.Reporter)
-      forward(parent)
+      :slave.relay(parent)
     end)
 
     multirpc(nodes, :slave, :pseudo, [node(), [__MODULE__.Reporter]])
     :ok
   end
 
-  defp forward(parent) do
-    receive do
-      msg -> send(parent, msg)
-    end
-
-    forward(parent)
-  end
-
   @doc """
   Sends a report back to the reporter process configured with `start_reporter/1`.
   """
diff --git a/test/support/test_helpers.ex b/test/support/test_helpers.ex
index e691177..9488abe 100644
--- a/test/support/test_helpers.ex
+++ b/test/support/test_helpers.ex
@@ -11,7 +11,7 @@ defmodule Firenest.TestHelpers do
   @doc """
   Waits until fun is true `count * 10` milliseconds.
   """
-  def wait_until(fun, count \\ 1000) do
+  def wait_until(fun, count \\ div(Application.fetch_env!(:ex_unit, :assert_receive_timeout), 10)) do
     cond do
       count == 0 ->
         raise "waited until fun returned true but it never did"

From 53890e9248cae4c2b9c909c39ca6843299854c8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 12 Jun 2018 21:07:36 +0200
Subject: [PATCH 14/40] Rename PG to ReplicatedState

---
 lib/firenest/{pg.ex => replicated_state.ex}   | 157 ++++++++++--------
 ...{pg_test.exs => replicated_state_test.exs} | 134 ++++++++-------
 2 files changed, 150 insertions(+), 141 deletions(-)
 rename lib/firenest/{pg.ex => replicated_state.ex} (67%)
 rename test/firenest/{pg_test.exs => replicated_state_test.exs} (51%)

diff --git a/lib/firenest/pg.ex b/lib/firenest/replicated_state.ex
similarity index 67%
rename from lib/firenest/pg.ex
rename to lib/firenest/replicated_state.ex
index f247ba3..28d0d2c 100644
--- a/lib/firenest/pg.ex
+++ b/lib/firenest/replicated_state.ex
@@ -1,8 +1,21 @@
-defmodule Firenest.PG do
+defmodule Firenest.ReplicatedState do
+  @moduledoc """
+  Facility for replicating ephemeral state across cluster.
+
+  Allows registering some state attached to aprocess and replicated
+  across cluster. The state is always linked to a liftetime of a
+  process - when the process dies the state will be removed on all
+  nodes and in case nodes get disconnected, all the state from
+  disconnected nodes will be (temporarily) removed. The state
+  can be only updated from the node where the process lives.
+
+  The state is managed through callbacks that are invoked on the
+  node where the process lives and remotely on other nodes.
+  """
   alias Firenest.SyncedServer
 
-  @type pg() :: atom()
-  @type group() :: term()
+  @type server() :: atom()
+  @type key() :: term()
   @type value() :: term()
 
   @doc """
@@ -16,24 +29,24 @@ defmodule Firenest.PG do
       nodes, defaults to 50 ms;
 
   """
-  defdelegate child_spec(opts), to: Firenest.PG.Supervisor
+  defdelegate child_spec(opts), to: Firenest.ReplicatedState.Supervisor
 
-  @spec join(pg(), group(), pid(), value()) :: :ok | {:error, :already_joined}
-  def join(pg, group, pid, value) when node(pid) == node() do
-    server = partition_info!(pg, group)
-    SyncedServer.call(server, {:join, group, pid, value})
+  @spec join(server(), key(), pid(), value()) :: :ok | {:error, :already_joined}
+  def join(server, key, pid, value) when node(pid) == node() do
+    partition = partition_info!(server, key)
+    SyncedServer.call(partition, {:join, key, pid, value})
   end
 
-  @spec leave(pg(), group(), pid()) :: :ok | {:error, :not_member}
-  def leave(pg, group, pid) when node(pid) == node() do
-    server = partition_info!(pg, group)
-    SyncedServer.call(server, {:leave, group, pid})
+  @spec leave(server(), key(), pid()) :: :ok | {:error, :not_member}
+  def leave(server, key, pid) when node(pid) == node() do
+    partition = partition_info!(server, key)
+    SyncedServer.call(partition, {:leave, key, pid})
   end
 
-  @spec leave(pg(), pid()) :: :ok | {:error, :not_member}
-  def leave(pg, pid) when node(pid) == node() do
-    servers = partition_infos!(pg)
-    replies = multicall(servers, {:leave, pid}, 5_000)
+  @spec leave(server(), pid()) :: :ok | {:error, :not_member}
+  def leave(server, pid) when node(pid) == node() do
+    partitions = partition_infos!(server)
+    replies = multicall(partitions, {:leave, pid}, 5_000)
 
     if :ok in replies do
       :ok
@@ -42,26 +55,26 @@ defmodule Firenest.PG do
     end
   end
 
-  @spec update(pg(), group(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
-  def update(pg, group, pid, update) when node(pid) == node() and is_function(update, 1) do
-    server = partition_info!(pg, group)
-    SyncedServer.call(server, {:update, group, pid, update})
+  @spec update(server(), key(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
+  def update(server, key, pid, update) when node(pid) == node() and is_function(update, 1) do
+    partition = partition_info!(server, key)
+    SyncedServer.call(partition, {:update, key, pid, update})
   end
 
-  @spec replace(pg(), group(), pid(), value()) :: :ok | {:error, :not_member}
-  def replace(pg, group, pid, value) when node(pid) == node() do
-    server = partition_info!(pg, group)
-    SyncedServer.call(server, {:replace, group, pid, value})
+  @spec replace(server(), key(), pid(), value()) :: :ok | {:error, :not_member}
+  def replace(server, key, pid, value) when node(pid) == node() do
+    partition = partition_info!(server, key)
+    SyncedServer.call(partition, {:replace, key, pid, value})
   end
 
-  @spec members(pg(), group()) :: [value()]
-  def members(pg, group) do
-    server = partition_info!(pg, group)
-    SyncedServer.call(server, {:members, group}).()
+  @spec members(server(), key()) :: [value()]
+  def members(server, key) do
+    partition = partition_info!(server, key)
+    SyncedServer.call(partition, {:members, key}).()
   end
 
   # TODO
-  # def dirty_list(pg, group)
+  # def dirty_list(server, group)
 
   defp multicall(servers, request, timeout) do
     servers
@@ -87,26 +100,26 @@ defmodule Firenest.PG do
     end)
   end
 
-  defp partition_info!(pg, group) do
-    hash = :erlang.phash2(group)
+  defp partition_info!(server, key) do
+    hash = :erlang.phash2(key)
     extract = {:element, {:+, {:rem, {:const, hash}, :"$1"}, 1}, :"$2"}
     ms = [{{:partitions, :"$1", :"$2"}, [], [extract]}]
-    [info] = :ets.select(pg, ms)
+    [info] = :ets.select(server, ms)
     info
   catch
     :error, :badarg ->
-      raise ArgumentError, "unknown group: #{inspect(pg)}"
+      raise ArgumentError, "unknown key: #{inspect(server)}"
   end
 
-  defp partition_infos!(pg) do
-    Tuple.to_list(:ets.lookup_element(pg, :partitions, 3))
+  defp partition_infos!(server) do
+    Tuple.to_list(:ets.lookup_element(server, :partitions, 3))
   catch
     :error, :badarg ->
-      raise ArgumentError, "unknown group: #{inspect(pg)}"
+      raise ArgumentError, "unknown group: #{inspect(server)}"
   end
 end
 
-defmodule Firenest.PG.Supervisor do
+defmodule Firenest.ReplicatedState.Supervisor do
   @moduledoc false
   use Supervisor
 
@@ -131,7 +144,7 @@ defmodule Firenest.PG.Supervisor do
 
     children =
       for name <- names,
-          do: {Firenest.PG.Server, {name, topology, opts}}
+          do: {Firenest.ReplicatedState.Server, {name, topology, opts}}
 
     :ets.new(name, [:named_table, :set, read_concurrency: true])
     :ets.insert(name, {:partitions, partitions, List.to_tuple(names)})
@@ -140,7 +153,7 @@ defmodule Firenest.PG.Supervisor do
   end
 end
 
-defmodule Firenest.PG.Server do
+defmodule Firenest.ReplicatedState.Server do
   @moduledoc false
   use Firenest.SyncedServer
 
@@ -178,25 +191,25 @@ defmodule Firenest.PG.Server do
   def handshake_data(%{clock: clock}), do: clock
 
   @impl true
-  def handle_call({:join, group, pid, value}, _from, state) do
+  def handle_call({:join, key, pid, value}, _from, state) do
     %{values: values, pids: pids} = state
     Process.link(pid)
-    key = {group, pid}
+    ets_key = {key, pid}
 
-    if :ets.member(values, key) do
+    if :ets.member(values, ets_key) do
       {:reply, {:error, :already_joined}, state}
     else
-      :ets.insert(values, {key, value})
-      :ets.insert(pids, key)
-      state = schedule_broadcast_events(state, [{:join, group, pid, value}])
+      :ets.insert(values, {ets_key, value})
+      :ets.insert(pids, ets_key)
+      state = schedule_broadcast_events(state, [{:join, key, pid, value}])
       {:reply, :ok, state}
     end
   end
 
-  def handle_call({:leave, group, pid}, _from, state) do
+  def handle_call({:leave, key, pid}, _from, state) do
     %{values: values, pids: pids} = state
-    key = {group, pid}
-    ms = [{key, [], [true]}]
+    ets_key = {key, pid}
+    ms = [{ets_key, [], [true]}]
 
     case :ets.select_delete(pids, ms) do
       0 ->
@@ -207,21 +220,21 @@ defmodule Firenest.PG.Server do
           Process.unlink(pid)
         end
 
-        :ets.delete(values, key)
-        state = schedule_broadcast_events(state, [{:leave, group, pid}])
+        :ets.delete(values, ets_key)
+        state = schedule_broadcast_events(state, [{:leave, key, pid}])
         {:reply, :ok, state}
     end
   end
 
-  def handle_call({:update, group, pid, update}, _from, state) do
+  def handle_call({:update, key, pid, update}, _from, state) do
     %{values: values} = state
-    key = {group, pid}
+    ets_key = {key, pid}
 
-    case ets_fetch_element(values, key, 2) do
+    case ets_fetch_element(values, ets_key, 2) do
       {:ok, value} ->
         new_value = update.(value)
-        :ets.insert(values, {key, new_value})
-        state = schedule_broadcast_events(state, [{:replace, group, pid, new_value}])
+        :ets.insert(values, {ets_key, new_value})
+        state = schedule_broadcast_events(state, [{:replace, key, pid, new_value}])
         {:reply, :ok, state}
 
       :error ->
@@ -229,13 +242,13 @@ defmodule Firenest.PG.Server do
     end
   end
 
-  def handle_call({:replace, group, pid, value}, _from, state) do
+  def handle_call({:replace, key, pid, value}, _from, state) do
     %{values: values} = state
-    key = {group, pid}
+    ets_key = {key, pid}
 
-    if :ets.member(values, key) do
-      :ets.insert(values, {key, value})
-      state = schedule_broadcast_events(state, [{:replace, group, pid, value}])
+    if :ets.member(values, ets_key) do
+      :ets.insert(values, {ets_key, value})
+      state = schedule_broadcast_events(state, [{:replace, key, pid, value}])
       {:reply, :ok, state}
     else
       {:reply, {:error, :not_member}, state}
@@ -256,12 +269,12 @@ defmodule Firenest.PG.Server do
     end
   end
 
-  def handle_call({:members, group}, _from, state) do
+  def handle_call({:members, key}, _from, state) do
     %{values: values} = state
 
     read = fn ->
-      local = {{{group, :_}, :"$1"}, [], [:"$1"]}
-      remote = {{{group, :_}, :_, :"$1"}, [], [:"$1"]}
+      local = {{{key, :_}, :"$1"}, [], [:"$1"]}
+      remote = {{{key, :_}, :_, :"$1"}, [], [:"$1"]}
       :ets.select(values, [local, remote])
     end
 
@@ -354,9 +367,9 @@ defmodule Firenest.PG.Server do
         :error
 
       list ->
-        ms = for key <- list, do: {{key, :_}, [], [true]}
+        ms = for ets_key <- list, do: {{ets_key, :_}, [], [true]}
         :ets.select_delete(values, ms)
-        leaves = for {group, pid} <- list, do: {:leave, group, pid}
+        leaves = for {key, pid} <- list, do: {:leave, key, pid}
         {:ok, leaves}
     end
   end
@@ -396,16 +409,16 @@ defmodule Firenest.PG.Server do
   defp handle_events(%{values: values} = state, from, events) do
     {joins, leaves} =
       Enum.reduce(events, {[], []}, fn
-        {:leave, group, pid}, {joins, leaves} ->
-          leave = {{{group, pid}, from, :_}, [], [true]}
+        {:leave, key, pid}, {joins, leaves} ->
+          leave = {{{key, pid}, from, :_}, [], [true]}
           {joins, [leave | leaves]}
 
-        {:replace, group, pid, value}, {joins, leaves} ->
-          join = {{group, pid}, from, value}
+        {:replace, key, pid, value}, {joins, leaves} ->
+          join = {{key, pid}, from, value}
           {[join | joins], leaves}
 
-        {:join, group, pid, value}, {joins, leaves} ->
-          join = {{group, pid}, from, value}
+        {:join, key, pid, value}, {joins, leaves} ->
+          join = {{key, pid}, from, value}
           {[join | joins], leaves}
       end)
 
@@ -420,7 +433,7 @@ defmodule Firenest.PG.Server do
   defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
     %{remote_clocks: remote_clocks} = state
     delete_ms = [{{:_, from, :_}, [], [true]}]
-    inserts = for {key, value} <- transfer, do: {key, from, value}
+    inserts = for {ets_key, value} <- transfer, do: {ets_key, from, value}
     :ets.select_delete(values, delete_ms)
     :ets.insert(values, inserts)
     %{state | remote_clocks: %{remote_clocks | from => clock}}
diff --git a/test/firenest/pg_test.exs b/test/firenest/replicated_state_test.exs
similarity index 51%
rename from test/firenest/pg_test.exs
rename to test/firenest/replicated_state_test.exs
index a9def0f..355c4dc 100644
--- a/test/firenest/pg_test.exs
+++ b/test/firenest/replicated_state_test.exs
@@ -1,8 +1,8 @@
-defmodule Firenest.PGTest do
+defmodule Firenest.ReplicatedStateTest do
   use ExUnit.Case, async: true
 
   alias Firenest.Topology, as: T
-  alias Firenest.PG
+  alias Firenest.ReplicatedState, as: R
 
   import Firenest.TestHelpers
 
@@ -11,33 +11,33 @@ defmodule Firenest.PGTest do
   end
 
   setup %{test: test, topology: topology} do
-    assert {:ok, _} = start_supervised({PG, name: test, topology: topology})
-    {:ok, pg: test}
+    assert {:ok, _} = start_supervised({R, name: test, topology: topology})
+    {:ok, server: test}
   end
 
   describe "join/5" do
-    test "adds process", %{pg: pg} do
-      assert PG.join(pg, :foo, self(), :baz) == :ok
-      assert [:baz] == PG.members(pg, :foo)
+    test "adds process", %{server: server} do
+      assert R.join(server, :foo, self(), :baz) == :ok
+      assert [:baz] == R.members(server, :foo)
     end
 
-    test "rejects double joins", %{pg: pg} do
-      assert PG.join(pg, :foo, self(), :baz) == :ok
-      assert PG.join(pg, :foo, self(), :baz) == {:error, :already_joined}
+    test "rejects double joins", %{server: server} do
+      assert R.join(server, :foo, self(), :baz) == :ok
+      assert R.join(server, :foo, self(), :baz) == {:error, :already_joined}
     end
 
-    test "cleans up entries after process dies", %{pg: pg} do
+    test "cleans up entries after process dies", %{server: server} do
       {pid, ref} = spawn_monitor(Process, :sleep, [:infinity])
-      PG.join(pg, :foo, pid, :baz)
-      assert [_] = PG.members(pg, :foo)
+      R.join(server, :foo, pid, :baz)
+      assert [_] = R.members(server, :foo)
       Process.exit(pid, :kill)
       assert_receive {:DOWN, ^ref, _, _, _}
-      assert [] = PG.members(pg, :foo)
+      assert [] = R.members(server, :foo)
     end
 
-    test "pg dies if other linked process dies", %{pg: pg} do
+    test "server dies if other linked process dies", %{server: server} do
       parent = self()
-      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(server, "Supervisor"))
       ref = Process.monitor(pid)
 
       temp =
@@ -55,86 +55,86 @@ defmodule Firenest.PGTest do
   end
 
   describe "leave/2" do
-    test "removes entry", %{pg: pg} do
-      PG.join(pg, :foo, self(), :baz)
+    test "removes entry", %{server: server} do
+      R.join(server, :foo, self(), :baz)
 
-      assert [_] = PG.members(pg, :foo)
-      assert PG.leave(pg, self()) == :ok
-      assert [] == PG.members(pg, :foo)
+      assert [_] = R.members(server, :foo)
+      assert R.leave(server, self()) == :ok
+      assert [] == R.members(server, :foo)
     end
 
-    test "does not remove non members", %{pg: pg} do
-      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+    test "does not remove non members", %{server: server} do
+      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(server, "Supervisor"))
       Process.link(pid)
 
-      assert PG.leave(pg, self()) == {:error, :not_member}
+      assert R.leave(server, self()) == {:error, :not_member}
       {:links, links} = Process.info(self(), :links)
       assert pid in links
     end
   end
 
   describe "leave/4" do
-    test "removes single entry", %{pg: pg} do
-      PG.join(pg, :foo, self(), :baz)
-      assert [_] = PG.members(pg, :foo)
+    test "removes single entry", %{server: server} do
+      R.join(server, :foo, self(), :baz)
+      assert [_] = R.members(server, :foo)
 
-      assert PG.leave(pg, :foo, self()) == :ok
-      assert [] == PG.members(pg, :foo)
+      assert R.leave(server, :foo, self()) == :ok
+      assert [] == R.members(server, :foo)
     end
 
-    test "leaves other entries intact", %{pg: pg} do
+    test "leaves other entries intact", %{server: server} do
       pid = spawn_link(fn -> Process.sleep(:infinity) end)
-      PG.join(pg, :foo, self(), :baz)
-      PG.join(pg, :foo, pid, :baaz)
-      assert [_, _] = PG.members(pg, :foo)
+      R.join(server, :foo, self(), :baz)
+      R.join(server, :foo, pid, :baaz)
+      assert [_, _] = R.members(server, :foo)
 
-      assert PG.leave(pg, :foo, self()) == :ok
-      assert [:baaz] == PG.members(pg, :foo)
+      assert R.leave(server, :foo, self()) == :ok
+      assert [:baaz] == R.members(server, :foo)
     end
 
-    test "does not remove non members", %{pg: pg} do
-      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(pg, "Supervisor"))
+    test "does not remove non members", %{server: server} do
+      [{_, pid, _, _}] = Supervisor.which_children(Module.concat(server, "Supervisor"))
       Process.link(pid)
 
-      assert PG.leave(pg, :foo, self()) == {:error, :not_member}
+      assert R.leave(server, :foo, self()) == {:error, :not_member}
       {:links, links} = Process.info(self(), :links)
       assert pid in links
     end
   end
 
   describe "update/5" do
-    test "executes the update if entry is present", %{pg: pg} do
+    test "executes the update if entry is present", %{server: server} do
       parent = self()
-      PG.join(pg, :foo, self(), 1)
-      assert [1] == PG.members(pg, :foo)
+      R.join(server, :foo, self(), 1)
+      assert [1] == R.members(server, :foo)
 
       update = fn value ->
         send(parent, value)
         value + 1
       end
 
-      assert PG.update(pg, :foo, self(), update) == :ok
+      assert R.update(server, :foo, self(), update) == :ok
       assert_received 1
-      assert [2] == PG.members(pg, :foo)
+      assert [2] == R.members(server, :foo)
     end
 
-    test "does not execute update if entry is absent", %{pg: pg} do
+    test "does not execute update if entry is absent", %{server: server} do
       parent = self()
       update = fn value -> Process.exit(parent, {:unexpected_update, value}) end
-      assert PG.update(pg, :foo, self(), update) == {:error, :not_member}
+      assert R.update(server, :foo, self(), update) == {:error, :not_member}
     end
   end
 
   describe "replace/5" do
-    test "updates value if entry is present", %{pg: pg} do
-      PG.join(pg, :foo, self(), 1)
-      assert [1] == PG.members(pg, :foo)
-      assert PG.replace(pg, :foo, self(), 2) == :ok
-      assert [2] == PG.members(pg, :foo)
+    test "updates value if entry is present", %{server: server} do
+      R.join(server, :foo, self(), 1)
+      assert [1] == R.members(server, :foo)
+      assert R.replace(server, :foo, self(), 2) == :ok
+      assert [2] == R.members(server, :foo)
     end
 
-    test "does not update value if entry is absent", %{pg: pg} do
-      assert PG.replace(pg, :foo, self(), 2) == {:error, :not_member}
+    test "does not update value if entry is absent", %{server: server} do
+      assert R.replace(server, :foo, self(), 2) == {:error, :not_member}
     end
   end
 
@@ -146,67 +146,63 @@ defmodule Firenest.PGTest do
       wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
       nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
       topology = Firenest.Test
-      pg = Firenest.Test.PG
-      %{start: start} = PG.child_spec(name: pg, topology: topology)
+      server = Firenest.Test.ReplicatedServer
+      %{start: start} = R.child_spec(name: server, topology: topology)
       Firenest.Test.start_link(nodes, start)
       nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
 
-      {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, pg: pg}
-    end
-
-    setup %{test: test} do
-      {:ok, group: test}
+      {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, server: server}
     end
 
     test "remote join is propagated", config do
-      %{pg: pg, group: group, nodes: [second | _]} = config
+      %{server: server, test: test, nodes: [second | _]} = config
 
       quote do
         spawn(fn ->
-          :ok = PG.join(unquote(pg), unquote(group), self(), :baz)
+          :ok = R.join(unquote(server), unquote(test), self(), :baz)
           :timer.sleep(:infinity)
         end)
       end
       |> eval_on_node(second, config)
 
-      wait_until(fn -> PG.members(pg, group) == [:baz] end)
+      wait_until(fn -> R.members(server, test) == [:baz] end)
     end
 
     test "propages changes when nodes were disconnected", config do
-      %{topology: topology, pg: pg, test: test, nodes: [second, third]} = config
+      %{topology: topology, server: server, test: test, nodes: [second, third]} = config
       Process.register(self(), test)
 
       quote do
         spawn(fn ->
           Process.register(self(), unquote(test))
-          :ok = PG.join(unquote(pg), unquote(test), self(), :baz)
+          :ok = R.join(unquote(server), unquote(test), self(), :baz)
           Process.sleep(:infinity)
         end)
       end
       |> eval_on_node(second, config)
 
-      quote(do: PG.members(unquote(pg), unquote(test)) == [:baz])
+      quote(do: R.members(unquote(server), unquote(test)) == [:baz])
       |> await_on_node(third, config)
 
       quote do
         T.disconnect(unquote(topology), elem(unquote(third), 0))
         pid = Process.whereis(unquote(test))
-        :ok = PG.leave(unquote(pg), unquote(test), pid)
+        :ok = R.leave(unquote(server), unquote(test), pid)
 
         spawn(fn ->
-          :ok = PG.join(unquote(pg), unquote(test), self(), :bar)
+          :ok = R.join(unquote(server), unquote(test), self(), :bar)
           Process.sleep(:infinity)
         end)
       end
       |> eval_on_node(second, config)
 
-      quote(do: PG.members(unquote(pg), unquote(test)) == [])
+      quote(do: R.members(unquote(server), unquote(test)) == [])
       |> await_on_node(third, config)
 
       quote(do: T.connect(unquote(topology), elem(unquote(third), 0)))
       |> eval_on_node(second, config)
 
-      quote(do: PG.members(unquote(pg), unquote(test)) == [:bar])
+      quote(do: R.members(unquote(server), unquote(test)) == [:bar])
       |> await_on_node(third, config)
     end
 

From 0421ca9652220c2ecbf2fb3d4c51c61cbb177648 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 12 Jun 2018 22:01:22 +0200
Subject: [PATCH 15/40] Docs for the replicated state server

---
 lib/firenest/replicated_state.ex | 162 +++++++++++++++++++++++++++----
 test/shared/test.ex              |   2 +-
 2 files changed, 146 insertions(+), 18 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 28d0d2c..bbd7a2f 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -16,33 +16,158 @@ defmodule Firenest.ReplicatedState do
 
   @type server() :: atom()
   @type key() :: term()
-  @type value() :: term()
+
+  @type local_delta() :: term()
+  @type remote_delta() :: term()
+  @type state() :: term()
+  @type config() :: term()
+
+  @doc """
+  Set up new stare for a process.
+
+  The `arg` is received from the corrsponding `join/4` call.
+  Sets up local data for incremental tracking of changes to state and
+  an initial state that will be replicated to remote nodes.
+  """
+  @callback local_join(arg :: term(), config()) ::
+              {initial_delta :: local_delta(), initial_state :: state()}
+
+  @doc """
+  Called whenever the `update/4` function is called for a process.
+
+  It returns updated local delta and updated state.
+
+  This is a good place for broadcasting local state changes.
+  """
+  @callback local_update(update :: term(), local_delta(), state(), config()) ::
+              {local_delta(), state()}
 
   @doc """
+  Called whenever a local process dies or the `leave/3` or `leave/2` function
+  is called for a process.
+
+  The return value is ignored.
+
+  This is a good place for broadcasting local state changes.
+  """
+  @callback local_leave(state(), config()) :: term()
+
+  @doc """
+  Called whenever the server is about to replicate state to a remote node.
+
+  It takes in the local delta value constructed in the `c:local_update/4`
+  calls and returns a remote delta value that will be replicated to other
+  servers.
+
+  In case the callback is not provided it defaults to just returning local delta.
+  """
+  @callback prepare_remote_delta(local_delta(), config()) :: remote_delta()
+
+  @doc """
+  Called whenever a remote delta is received from another node.
+
+  The `remote_delta` value is the return value of `c:prepare_remote_delta/2`
+  callback. The result of applying the remote delta to state must be
+  extactly the same as the result of applying local updates to the
+  state in the `c:local_update/3` callback.
+  """
+  @callback handle_remote_delta(remote_delta(), state(), config()) :: {:ok, state()} | :noop
+
+  @doc """
+  Called when remote changes are received by the local process.
+
+  It receives a list of observed remote changes for all the tracked keys.
+  A `process_state_change` specifies how the state for a single process changed.
+  In particular the change can be:
+
+    * `{initial_state, [:joined, ...]}` in case the process just joined
+    * `{last_known_state, [..., :left]}` in case the process just left
+    * `{last_known_state, [{:replace, state}, ...]}` in case it was not possible
+      to replicate incremental changes to remote state through deltas and a full
+      state transfer was performed.
+    * `{last_known_state, [..., {:delta, remote_delta}, ...]}` in case
+      incremental changes to the remote state were communicated through the delta
+      mechanism.
+
+  This callback is optional and its behaviour depends on the value
+  of the `:remote_changes` option when starting the process.
+
+    * `:ignore` - the callback is not invoked and the server skips
+      all operations required for tracking the changes. This is the
+      default and should be chosen if precise state change tracking
+      is not required.
+
+    * `:observe_full` - calls the callback with as precise information
+      about the remote state changes as possible. This means that, for
+      example, for a very short-lived process a `[:joined, :left]` sequence
+      of changes is possible.
+
+    * `:observe_collapsed` - collapses the information about remote state
+      changes. For example a `[:joined, {:delta, ...}]` sequence is collapsed
+      into just `[:joined]` with all the state changes alrady applied and a
+      `[{:delta, ...}, :left]` sequence is collapsed into just `[:left]`.
+      This also means that for short-lived processes, no information
+      may be transfered if the chnages would contain both `:joined` and `:left`.
+
+  This is a good place for broadcasting remote state changes.
+
+  The return value is ignored.
+  """
+  @callback observe_remote_changes(observed_remote_changes, config()) :: term()
+            when observed_remote_changes: [{key(), [process_state_change]}],
+                 process_state_change: {current_state :: state(), [change]},
+                 change:
+                   :joined | {:replace, new_state :: state()} | {:delta, remote_delta()} | :left
+
+  @optional_callbacks [observe_remote_changes: 2, prepare_remote_delta: 2]
+
+  @doc """
+  Returns a child spec for the replicated state server.
 
   ## Options
 
     * `:name` - name for the process, required;
     * `:topology` - name of the supporting topology, required;
     * `:partitions` - number of partitions, defaults to 1;
-    * `:broadcast_timeout` - delay of broadcasting local events to other
-      nodes, defaults to 50 ms;
+    * `:broadcast_timeout` - delay of broadcasting local events to other nodes,
+      defaults to 50 ms;
+    * `:remote_changes` - specifies behaviour of the `c:observe_remote_changes/2`
+      callback and can be `:observe_full | :observe_collapsed | :ignore`,
+      defaults to `:ignore`;
+    * `:config` - constant value passed to all callbacks, defaults to `nil`;
 
   """
   defdelegate child_spec(opts), to: Firenest.ReplicatedState.Supervisor
 
-  @spec join(server(), key(), pid(), value()) :: :ok | {:error, :already_joined}
-  def join(server, key, pid, value) when node(pid) == node() do
+  @doc """
+  Registeres process `pid` under `key` and starts tracking its state.
+
+  This calls the `c:local_join/2` callback with `arg` inside the server.
+  """
+  @spec join(server(), key(), pid(), term()) :: :ok | {:error, :already_joined}
+  def join(server, key, pid, arg) when node(pid) == node() do
     partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:join, key, pid, value})
+    SyncedServer.call(partition, {:join, key, pid, arg})
   end
 
+  @doc """
+  Unregisteres process `pid` from `key` and removes its state.
+
+  This calls the `c:local_leave/2` callback inside the server.
+  """
   @spec leave(server(), key(), pid()) :: :ok | {:error, :not_member}
   def leave(server, key, pid) when node(pid) == node() do
     partition = partition_info!(server, key)
     SyncedServer.call(partition, {:leave, key, pid})
   end
 
+  @doc """
+  Unregisteres process `pid` from all keys it's registered under and
+  removes all its state.
+
+  This calls the `c:local_leave/2` callback inside the server for
+  each key the proess is leaving.
+  """
   @spec leave(server(), pid()) :: :ok | {:error, :not_member}
   def leave(server, pid) when node(pid) == node() do
     partitions = partition_infos!(server)
@@ -55,22 +180,25 @@ defmodule Firenest.ReplicatedState do
     end
   end
 
-  @spec update(server(), key(), pid(), (value() -> value())) :: :ok | {:error, :not_member}
-  def update(server, key, pid, update) when node(pid) == node() and is_function(update, 1) do
-    partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:update, key, pid, update})
-  end
+  @doc """
+  Updates state of process `pid` under `key`.
 
-  @spec replace(server(), key(), pid(), value()) :: :ok | {:error, :not_member}
-  def replace(server, key, pid, value) when node(pid) == node() do
+  This calls the `c:local_update/4` callback inside the server passing
+  the value of `update`.
+  """
+  @spec update(server(), key(), pid(), term()) :: :ok | {:error, :not_member}
+  def update(server, key, pid, update) when node(pid) == node() do
     partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:replace, key, pid, value})
+    SyncedServer.call(partition, {:update, key, pid, update})
   end
 
-  @spec members(server(), key()) :: [value()]
-  def members(server, key) do
+  @doc """
+  Lists all registered states for a given `key`.
+  """
+  @spec list(server(), key()) :: [state()]
+  def list(server, key) do
     partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:members, key}).()
+    SyncedServer.call(partition, {:list, key}).()
   end
 
   # TODO
diff --git a/test/shared/test.ex b/test/shared/test.ex
index 4dfd3eb..e37f181 100644
--- a/test/shared/test.ex
+++ b/test/shared/test.ex
@@ -28,7 +28,7 @@ defmodule Firenest.Test do
     catch
       kind, reason ->
         exception = Exception.format(kind, reason, System.stacktrace())
-        Logger.error("Eval failed on node #{inspect node()}\n#{exception}")
+        Logger.error("Eval failed on node #{inspect(node())}\n#{exception}")
         {:noreply, state}
     end
 

From ccd107caa686a66d6e254aa89b8a3869bdd63c58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 12 Jun 2018 22:20:15 +0200
Subject: [PATCH 16/40] Doc updates

---
 lib/firenest/replicated_state.ex | 60 ++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index bbd7a2f..7032113 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -2,15 +2,25 @@ defmodule Firenest.ReplicatedState do
   @moduledoc """
   Facility for replicating ephemeral state across cluster.
 
-  Allows registering some state attached to aprocess and replicated
-  across cluster. The state is always linked to a liftetime of a
-  process - when the process dies the state will be removed on all
-  nodes and in case nodes get disconnected, all the state from
-  disconnected nodes will be (temporarily) removed. The state
-  can be only updated from the node where the process lives.
+  Allows registering an ephemeral state attached to a process
+  that will be replicated across the cluster. The state is always
+  linked to a lifetime of a process - when the process dies the
+  state will be removed on all nodes and in case nodes get disconnected,
+  all the state from disconnected nodes will be (temporarily) removed.
+  The state can be only modified from the node where the process lives.
 
   The state is managed through callbacks that are invoked on the
-  node where the process lives and remotely on other nodes.
+  node where the process lives and remotely on other nodes when local
+  changes are propagated.
+
+  The state is replicated incrementally through building local "deltas"
+  (or changes to state) and periodically replicating them remotely. Only
+  some amount of recent deltas is retained for "catching up" remote nodes.
+  If remote node is too far behind the current state, a full state transfer
+  will be transferred to update it. If keeping track of incremental changes
+  is not convenient for a particular state type, the value of a delta can
+  be set to be equal to the current state - this will always cause full
+  state transfers.
   """
   alias Firenest.SyncedServer
 
@@ -25,15 +35,18 @@ defmodule Firenest.ReplicatedState do
   @doc """
   Set up new stare for a process.
 
-  The `arg` is received from the corrsponding `join/4` call.
+  The `arg` is received from the corresponding `join/4` call.
   Sets up local data for incremental tracking of changes to state and
   an initial state that will be replicated to remote nodes.
+
+  This is a good place for broadcasting local state changes.
   """
   @callback local_join(arg :: term(), config()) ::
               {initial_delta :: local_delta(), initial_state :: state()}
 
   @doc """
-  Called whenever the `update/4` function is called for a process.
+  Called whenever the `update/4` function is called to update
+  the state registered for a process.
 
   It returns updated local delta and updated state.
 
@@ -53,11 +66,13 @@ defmodule Firenest.ReplicatedState do
   @callback local_leave(state(), config()) :: term()
 
   @doc """
-  Called whenever the server is about to replicate state to a remote node.
+  Called whenever the server is about to incrementally replicate local
+  state to a remote node.
 
   It takes in the local delta value constructed in the `c:local_update/4`
   calls and returns a remote delta value that will be replicated to other
-  servers.
+  servers. The value of the local delta is reset to the initial delta value
+  returned from the `c:local_join/2` callback.
 
   In case the callback is not provided it defaults to just returning local delta.
   """
@@ -66,15 +81,15 @@ defmodule Firenest.ReplicatedState do
   @doc """
   Called whenever a remote delta is received from another node.
 
-  The `remote_delta` value is the return value of `c:prepare_remote_delta/2`
+  The `remote_delta` value is the return value of the `c:prepare_remote_delta/2`
   callback. The result of applying the remote delta to state must be
-  extactly the same as the result of applying local updates to the
+  exactly the same as the result of applying local updates to the
   state in the `c:local_update/3` callback.
   """
   @callback handle_remote_delta(remote_delta(), state(), config()) :: {:ok, state()} | :noop
 
   @doc """
-  Called when remote changes are received by the local process.
+  Called when remote changes are received by the local server.
 
   It receives a list of observed remote changes for all the tracked keys.
   A `process_state_change` specifies how the state for a single process changed.
@@ -86,11 +101,10 @@ defmodule Firenest.ReplicatedState do
       to replicate incremental changes to remote state through deltas and a full
       state transfer was performed.
     * `{last_known_state, [..., {:delta, remote_delta}, ...]}` in case
-      incremental changes to the remote state were communicated through the delta
-      mechanism.
+      incremental changes to the remote state were communicated.
 
   This callback is optional and its behaviour depends on the value
-  of the `:remote_changes` option when starting the process.
+  of the `:remote_changes` option provided when the server is started.
 
     * `:ignore` - the callback is not invoked and the server skips
       all operations required for tracking the changes. This is the
@@ -104,10 +118,10 @@ defmodule Firenest.ReplicatedState do
 
     * `:observe_collapsed` - collapses the information about remote state
       changes. For example a `[:joined, {:delta, ...}]` sequence is collapsed
-      into just `[:joined]` with all the state changes alrady applied and a
+      into just `[:joined]` with all the state changes already applied and a
       `[{:delta, ...}, :left]` sequence is collapsed into just `[:left]`.
       This also means that for short-lived processes, no information
-      may be transfered if the chnages would contain both `:joined` and `:left`.
+      may be transferred if the changes would contain both `:joined` and `:left`.
 
   This is a good place for broadcasting remote state changes.
 
@@ -140,7 +154,7 @@ defmodule Firenest.ReplicatedState do
   defdelegate child_spec(opts), to: Firenest.ReplicatedState.Supervisor
 
   @doc """
-  Registeres process `pid` under `key` and starts tracking its state.
+  Registers state for `pid` under `key`.
 
   This calls the `c:local_join/2` callback with `arg` inside the server.
   """
@@ -163,10 +177,10 @@ defmodule Firenest.ReplicatedState do
 
   @doc """
   Unregisteres process `pid` from all keys it's registered under and
-  removes all its state.
+  removes all its states.
 
   This calls the `c:local_leave/2` callback inside the server for
-  each key the proess is leaving.
+  each key the process is leaving.
   """
   @spec leave(server(), pid()) :: :ok | {:error, :not_member}
   def leave(server, pid) when node(pid) == node() do
@@ -181,7 +195,7 @@ defmodule Firenest.ReplicatedState do
   end
 
   @doc """
-  Updates state of process `pid` under `key`.
+  Updates state registered for process `pid` under `key`.
 
   This calls the `c:local_update/4` callback inside the server passing
   the value of `update`.

From 2b333d998c15771e6df9389f78ed30eea29966d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 13 Jun 2018 14:13:24 +0200
Subject: [PATCH 17/40] Update docs for ReplicatedState

---
 lib/firenest/replicated_state.ex | 144 +++++++++++++++++++------------
 1 file changed, 88 insertions(+), 56 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 7032113..1809533 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -1,26 +1,26 @@
 defmodule Firenest.ReplicatedState do
   @moduledoc """
-  Facility for replicating ephemeral state across cluster.
+  Distributed key-value store for ephemeral data.
 
-  Allows registering an ephemeral state attached to a process
-  that will be replicated across the cluster. The state is always
-  linked to a lifetime of a process - when the process dies the
-  state will be removed on all nodes and in case nodes get disconnected,
-  all the state from disconnected nodes will be (temporarily) removed.
-  The state can be only modified from the node where the process lives.
+  The key-value pairs are always attached to a lifetime of a
+  process and the state is replicated to all nodes across the
+  topology. When the attached process dies the state will be
+  removed on all nodes and in case the all the state from
+  disconnected nodes will be (temporarily) removed. The state
+  can only be attached to local processes.
 
   The state is managed through callbacks that are invoked on the
   node where the process lives and remotely on other nodes when local
   changes are propagated.
 
   The state is replicated incrementally through building local "deltas"
-  (or changes to state) and periodically replicating them remotely. Only
-  some amount of recent deltas is retained for "catching up" remote nodes.
-  If remote node is too far behind the current state, a full state transfer
-  will be transferred to update it. If keeping track of incremental changes
-  is not convenient for a particular state type, the value of a delta can
-  be set to be equal to the current state - this will always cause full
-  state transfers.
+  (changes to the state) and periodically replicating them remotely. Only
+  some amount of recent deltas are retained for "catching up" remote nodes.
+  If remote node is too far behind the current state and a delta-based
+  catch-up can't be performed, the server falls back to transferring the entire
+  local state. If keeping track of incremental changes is not convenient
+  for a particular state type, the value of a delta can be set to be equal
+  to the current state - this will always cause full state transfers.
   """
   alias Firenest.SyncedServer
 
@@ -32,38 +32,60 @@ defmodule Firenest.ReplicatedState do
   @type state() :: term()
   @type config() :: term()
 
+  @type delayed_update() :: {timeout :: pos_integer(), update :: term()}
+
+  @doc """
+  Called when a partition starts up.
+
+  It returns the initial delta value used for incremental state change
+  tracking and  an immutable `config` value that will be passed to all
+  callbacks.
+  """
+  @callback init(opts :: keyword()) :: {initial_delta :: local_delta(), config()}
+
   @doc """
-  Set up new stare for a process.
+  Called whenever the `put/4` function is called to create a new state.
 
-  The `arg` is received from the corresponding `join/4` call.
-  Sets up local data for incremental tracking of changes to state and
-  an initial state that will be replicated to remote nodes.
+  The `arg` is received from the corresponding `put/4` call.
+  For explanation of the `delayed_update` return value see the
+  `c:local_update/4` callback.
 
   This is a good place for broadcasting local state changes.
   """
-  @callback local_join(arg :: term(), config()) ::
-              {initial_delta :: local_delta(), initial_state :: state()}
+  @callback local_put(arg :: term(), config()) ::
+              {:ok, initial_state :: state()}
+              | {:ok, initial_state :: state(), delayed_update()}
 
   @doc """
-  Called whenever the `update/4` function is called to update
-  the state registered for a process.
+  Called whenever the `update/4` function is called to update a state.
 
-  It returns updated local delta and updated state.
+  It returns updated local delta and updated state. It can also return
+  a `{:delete, state}` tuple to indicate the state should be deleted.
+  This will trigger a call to the `c:local_delete/2` callback.
 
   This is a good place for broadcasting local state changes.
+
+  ## Delayed update
+
+  If the function returns a fourth element in the `:updated` tuple
+  consisting of `{timeout, update}`, a timer is started by the server
+  and the `c:local_update/4` callback will be called with the `update`
+  value after `timeout` milliseconds.
   """
   @callback local_update(update :: term(), local_delta(), state(), config()) ::
-              {local_delta(), state()}
+              {:updated, local_delta(), state()}
+              | {:updated, local_delta(), state(), delayed_update()}
+              | {:delete, state()}
 
   @doc """
-  Called whenever a local process dies or the `leave/3` or `leave/2` function
-  is called for a process.
+  Called whenever attached process dies or the `delete/3` or `delete/2`
+  function is called and state is about to be deleted.
 
   The return value is ignored.
 
   This is a good place for broadcasting local state changes.
   """
-  @callback local_leave(state(), config()) :: term()
+  @callback local_delete(state(), config()) :: term()
 
   @doc """
   Called whenever the server is about to incrementally replicate local
@@ -95,8 +117,8 @@ defmodule Firenest.ReplicatedState do
   A `process_state_change` specifies how the state for a single process changed.
   In particular the change can be:
 
-    * `{initial_state, [:joined, ...]}` in case the process just joined
-    * `{last_known_state, [..., :left]}` in case the process just left
+    * `{initial_state, [:put, ...]}` in case the process just joined
+    * `{last_known_state, [..., :delete]}` in case the process just left
     * `{last_known_state, [{:replace, state}, ...]}` in case it was not possible
       to replicate incremental changes to remote state through deltas and a full
       state transfer was performed.
@@ -113,15 +135,15 @@ defmodule Firenest.ReplicatedState do
 
     * `:observe_full` - calls the callback with as precise information
       about the remote state changes as possible. This means that, for
-      example, for a very short-lived process a `[:joined, :left]` sequence
+      example, for a very short-lived process a `[:put, :delete]` sequence
       of changes is possible.
 
     * `:observe_collapsed` - collapses the information about remote state
-      changes. For example a `[:joined, {:delta, ...}]` sequence is collapsed
-      into just `[:joined]` with all the state changes already applied and a
-      `[{:delta, ...}, :left]` sequence is collapsed into just `[:left]`.
+      changes. For example a `[:put, {:delta, ...}]` sequence is collapsed
+      into just `[:put]` with all the state changes already applied and a
+      `[{:delta, ...}, :delete]` sequence is collapsed into just `[:delete]`.
       This also means that for short-lived processes, no information
-      may be transferred if the changes would contain both `:joined` and `:left`.
+      may be transferred if the changes would contain both `:put` and `:delete`.
 
   This is a good place for broadcasting remote state changes.
 
@@ -131,13 +153,16 @@ defmodule Firenest.ReplicatedState do
             when observed_remote_changes: [{key(), [process_state_change]}],
                  process_state_change: {current_state :: state(), [change]},
                  change:
-                   :joined | {:replace, new_state :: state()} | {:delta, remote_delta()} | :left
+                   :put | {:replace, new_state :: state()} | {:delta, remote_delta()} | :delete
 
   @optional_callbacks [observe_remote_changes: 2, prepare_remote_delta: 2]
 
   @doc """
   Returns a child spec for the replicated state server.
 
+  Once started, each partition will call the `c:init/1` callback
+  passing all the provided options.
+
   ## Options
 
     * `:name` - name for the process, required;
@@ -148,44 +173,49 @@ defmodule Firenest.ReplicatedState do
     * `:remote_changes` - specifies behaviour of the `c:observe_remote_changes/2`
       callback and can be `:observe_full | :observe_collapsed | :ignore`,
       defaults to `:ignore`;
-    * `:config` - constant value passed to all callbacks, defaults to `nil`;
 
   """
   defdelegate child_spec(opts), to: Firenest.ReplicatedState.Supervisor
 
   @doc """
-  Registers state for `pid` under `key`.
+  Puts new state under `key` attached to lifetime of `pid`.
+
+  A special value `:partition` can be used for `pid` to indicate that the
+  lifetime of the value will be attached to the partition itself and
+  should be managed manually through `delete/3`, `delete/2` and `update/4`
+  calls with `:delete` returns from the `c:local_update/4` callback.
 
-  This calls the `c:local_join/2` callback with `arg` inside the server.
+  This calls the `c:local_put/2` callback with `arg` inside the server.
   """
-  @spec join(server(), key(), pid(), term()) :: :ok | {:error, :already_joined}
-  def join(server, key, pid, arg) when node(pid) == node() do
+  @spec put(server(), key(), pid() | :partition, term()) :: :ok | {:error, :already_joined}
+  def put(server, key, pid, arg) when pid == :parittion or node(pid) == node() do
     partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:join, key, pid, arg})
+    SyncedServer.call(partition, {:put, key, pid, arg})
   end
 
   @doc """
-  Unregisteres process `pid` from `key` and removes its state.
+  Deletes state under `key` attached to lifetime of `pid`.
 
-  This calls the `c:local_leave/2` callback inside the server.
+  For the explanation of `:partition` value for `pid` see `put/4`.
+
+  This calls the `c:local_delete/2` callback inside the server.
   """
-  @spec leave(server(), key(), pid()) :: :ok | {:error, :not_member}
-  def leave(server, key, pid) when node(pid) == node() do
+  @spec delete(server(), key(), pid() | :partition) :: :ok | {:error, :not_member}
+  def delete(server, key, pid) when pid == :partition or node(pid) == node() do
     partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:leave, key, pid})
+    SyncedServer.call(partition, {:delete, key, pid})
   end
 
   @doc """
-  Unregisteres process `pid` from all keys it's registered under and
-  removes all its states.
+  Deletes state under all keys attached to lifetime of `pid`.
 
-  This calls the `c:local_leave/2` callback inside the server for
+  This calls the `c:local_delete/2` callback inside the server for
   each key the process is leaving.
   """
-  @spec leave(server(), pid()) :: :ok | {:error, :not_member}
-  def leave(server, pid) when node(pid) == node() do
+  @spec delete(server(), pid()) :: :ok | {:error, :not_member}
+  def delete(server, pid) when node(pid) == node() do
     partitions = partition_infos!(server)
-    replies = multicall(partitions, {:leave, pid}, 5_000)
+    replies = multicall(partitions, {:delete, pid}, 5_000)
 
     if :ok in replies do
       :ok
@@ -195,19 +225,21 @@ defmodule Firenest.ReplicatedState do
   end
 
   @doc """
-  Updates state registered for process `pid` under `key`.
+  Updates state under `key` attached to lifetime of `pid`.
+
+  For the explanation of `:partition` value for `pid` see `put/4`.
 
   This calls the `c:local_update/4` callback inside the server passing
   the value of `update`.
   """
-  @spec update(server(), key(), pid(), term()) :: :ok | {:error, :not_member}
-  def update(server, key, pid, update) when node(pid) == node() do
+  @spec update(server(), key(), pid() | :partition, term()) :: :ok | {:error, :not_member}
+  def update(server, key, pid, update) when pid == :partition or node(pid) == node() do
     partition = partition_info!(server, key)
     SyncedServer.call(partition, {:update, key, pid, update})
   end
 
   @doc """
-  Lists all registered states for a given `key`.
+  Lists all states present for a given `key`.
   """
   @spec list(server(), key()) :: [state()]
   def list(server, key) do

From 6e9b92c768959f1fadf1447e3e42ec890789213a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 13 Jun 2018 14:28:03 +0200
Subject: [PATCH 18/40] Change how deleyed_update and delete work

---
 lib/firenest/replicated_state.ex | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 1809533..f42fcba 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -46,36 +46,33 @@ defmodule Firenest.ReplicatedState do
   @doc """
   Called whenever the `put/4` function is called to create a new state.
 
-  The `arg` is received from the corresponding `put/4` call.
-  For explanation of the `delayed_update` return value see the
-  `c:local_update/4` callback.
+  The `arg` is received from the corresponding `put/4` call. For the
+  explanation of the `delayed_update` and `:delete` return values see
+  the `c:local_update/4` callback.
 
   This is a good place for broadcasting local state changes.
   """
   @callback local_put(arg :: term(), config()) ::
-              {:ok, initial_state :: state()}
-              | {:ok, initial_state :: state(), delayed_update()}
+              {:ok, initial_state} | {:ok, initial_state, delayed_update() | :delete}
+            when initial_state: state
 
   @doc """
   Called whenever the `update/4` function is called to update a state.
 
-  It returns updated local delta and updated state. It can also return
-  a `{:delete, state}` tuple to indicate the state should be deleted.
-  This will trigger a call to the `c:local_delete/2` callback.
-
   This is a good place for broadcasting local state changes.
 
-  ## Delayed update
+  ## Delayed update and delete
+
+  If the function returns a third element in the tuple consisting of
+  `{timeout, update}`, a timer is started by the server and the
+  `c:local_update/4` callback will be called with the `update` value
+  after `timeout` milliseconds.
 
-  If the function returns a fourth element in the `:updated` tuple
-  consisting of `{timeout, update}`, a timer is started by the server
-  and the `c:local_update/4` callback will be called with the `update`
-  value after `timeout` milliseconds.
+  If the third element is `:delete`, the state will be immediately
+  deleted and the `c:local_delete/2` callback triggered.
   """
   @callback local_update(update :: term(), local_delta(), state(), config()) ::
-              {:updated, local_delta(), state()}
-              | {:updated, local_delta(), state(), delayed_update()}
-              | {:delete, state()}
+              {local_delta(), state()} | {local_delta(), state(), delayed_update() | :delete}
 
   @doc """
   Called whenever attached process dies or the `delete/3` or `delete/2`

From 5ff41b71e133e5cf2f7b19dec9d12277b41273ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 7 Aug 2018 14:50:53 +0200
Subject: [PATCH 19/40] Local replicated state tests

---
 lib/firenest/replicated_state.ex        | 295 ++++++++++++--------
 lib/firenest/replicated_state/store.ex  |  84 ++++++
 test/firenest/pub_sub_test.exs          |   2 +-
 test/firenest/replicated_state_test.exs | 350 ++++++++++++++++--------
 test/firenest/synced_server_test.exs    |   2 +-
 test/support/eval_state.ex              |  20 ++
 6 files changed, 516 insertions(+), 237 deletions(-)
 create mode 100644 lib/firenest/replicated_state/store.ex
 create mode 100644 test/support/eval_state.ex

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index f42fcba..10b54de 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -32,29 +32,31 @@ defmodule Firenest.ReplicatedState do
   @type state() :: term()
   @type config() :: term()
 
-  @type delayed_update() :: {timeout :: pos_integer(), update :: term()}
+  @type extra_action :: :delete | {:update_after, update :: term(), time :: pos_integer()}
 
   @doc """
   Called when a partition starts up.
 
-  It returns the initial delta value used for incremental state change
-  tracking and  an immutable `config` value that will be passed to all
-  callbacks.
+  It returns an immutable `config` value that will be passed to
+  all callbacks.
   """
+  # TODO: initial delta is returned here and passed to local_put.
+  # The reason: when we reset the local delta after broadcast, we'd
+  # need to store the initial_delta for each data point separately
   @callback init(opts :: keyword()) :: {initial_delta :: local_delta(), config()}
 
   @doc """
   Called whenever the `put/4` function is called to create a new state.
 
   The `arg` is received from the corresponding `put/4` call. For the
-  explanation of the `delayed_update` and `:delete` return values see
-  the `c:local_update/4` callback.
+  explanation of the `extra_action` return values see the
+  `c:local_update/4` callback.
 
   This is a good place for broadcasting local state changes.
   """
   @callback local_put(arg :: term(), config()) ::
-              {:ok, initial_state} | {:ok, initial_state, delayed_update() | :delete}
-            when initial_state: state
+              {initial_delta, initial_state} | {initial_delta, initial_state, extra_action()}
+            when initial_delta: local_delta(), initial_state: state()
 
   @doc """
   Called whenever the `update/4` function is called to update a state.
@@ -64,15 +66,15 @@ defmodule Firenest.ReplicatedState do
   ## Delayed update and delete
 
   If the function returns a third element in the tuple consisting of
-  `{timeout, update}`, a timer is started by the server and the
-  `c:local_update/4` callback will be called with the `update` value
-  after `timeout` milliseconds.
+  `{:update_after, update, time, update}`, a timer is started by the
+  server and the `c:local_update/4` callback will be called with the
+  `update` value after `time` milliseconds.
 
   If the third element is `:delete`, the state will be immediately
   deleted and the `c:local_delete/2` callback triggered.
   """
   @callback local_update(update :: term(), local_delta(), state(), config()) ::
-              {local_delta(), state()} | {local_delta(), state(), delayed_update() | :delete}
+              {local_delta(), state()} | {local_delta(), state(), extra_action()}
 
   @doc """
   Called whenever attached process dies or the `delete/3` or `delete/2`
@@ -105,7 +107,7 @@ defmodule Firenest.ReplicatedState do
   exactly the same as the result of applying local updates to the
   state in the `c:local_update/3` callback.
   """
-  @callback handle_remote_delta(remote_delta(), state(), config()) :: {:ok, state()} | :noop
+  @callback handle_remote_delta(remote_delta(), state(), config()) :: state()
 
   @doc """
   Called when remote changes are received by the local server.
@@ -182,7 +184,7 @@ defmodule Firenest.ReplicatedState do
   should be managed manually through `delete/3`, `delete/2` and `update/4`
   calls with `:delete` returns from the `c:local_update/4` callback.
 
-  This calls the `c:local_put/2` callback with `arg` inside the server.
+  This calls the `c:local_put/3` callback with `arg` inside the server.
   """
   @spec put(server(), key(), pid() | :partition, term()) :: :ok | {:error, :already_joined}
   def put(server, key, pid, arg) when pid == :parittion or node(pid) == node() do
@@ -298,8 +300,9 @@ defmodule Firenest.ReplicatedState.Supervisor do
     partitions = Keyword.get(opts, :partitions, 1)
     name = Keyword.fetch!(opts, :name)
     topology = Keyword.fetch!(opts, :topology)
+    handler = Keyword.fetch!(opts, :handler)
     supervisor = Module.concat(name, "Supervisor")
-    arg = {partitions, name, topology, opts}
+    arg = {partitions, name, topology, handler, opts}
 
     %{
       id: __MODULE__,
@@ -308,14 +311,14 @@ defmodule Firenest.ReplicatedState.Supervisor do
     }
   end
 
-  def init({partitions, name, topology, opts}) do
+  def init({partitions, name, topology, handler, opts}) do
     names =
       for partition <- 0..(partitions - 1),
           do: Module.concat(name, "Partition" <> Integer.to_string(partition))
 
     children =
       for name <- names,
-          do: {Firenest.ReplicatedState.Server, {name, topology, opts}}
+          do: {Firenest.ReplicatedState.Server, {name, topology, handler, opts}}
 
     :ets.new(name, [:named_table, :set, read_concurrency: true])
     :ets.insert(name, {:partitions, partitions, List.to_tuple(names)})
@@ -329,27 +332,30 @@ defmodule Firenest.ReplicatedState.Server do
   use Firenest.SyncedServer
 
   alias Firenest.SyncedServer
+  alias Firenest.ReplicatedState.Store
 
-  def child_spec({name, topology, opts}) do
+  def child_spec({name, topology, handler, opts}) do
     server_opts = [name: name, topology: topology]
 
     %{
       id: name,
-      start: {SyncedServer, :start_link, [__MODULE__, {name, opts}, server_opts]}
+      start: {SyncedServer, :start_link, [__MODULE__, {name, handler, opts}, server_opts]}
     }
   end
 
   @impl true
-  def init({name, opts}) do
+  def init({name, handler, opts}) do
     Process.flag(:trap_exit, true)
-    values = :ets.new(name, [:named_table, :protected, :ordered_set])
-    pids = :ets.new(__MODULE__.Pids, [:duplicate_bag, keypos: 2])
+    store = Store.new(name)
     broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
 
+    config = handler.init(opts)
+
     {:ok,
      %{
-       values: ets_whereis(values),
-       pids: pids,
+       store: store,
+       config: config,
+       handler: handler,
        broadcast_timer: nil,
        broadcast_timeout: broadcast_timeout,
        clock: 0,
@@ -362,107 +368,114 @@ defmodule Firenest.ReplicatedState.Server do
   def handshake_data(%{clock: clock}), do: clock
 
   @impl true
-  def handle_call({:join, key, pid, value}, _from, state) do
-    %{values: values, pids: pids} = state
-    Process.link(pid)
-    ets_key = {key, pid}
+  def handle_call({:put, key, pid, arg}, _from, state) do
+    %{store: store} = state
+
+    link(pid)
+
+    unless Store.present?(store, key, pid) do
+      case local_put(arg, key, pid, state) do
+        {:put, value, delta, state} ->
+          store = Store.local_put(store, key, pid, value, delta)
+          {:reply, :ok, %{state | store: store}}
 
-    if :ets.member(values, ets_key) do
-      {:reply, {:error, :already_joined}, state}
+        {:delete, value, _delta} ->
+          # TODO: this delta has to propagate remotely before delete
+          state = local_delete([value], state)
+          {:reply, :ok, state}
+      end
     else
-      :ets.insert(values, {ets_key, value})
-      :ets.insert(pids, ets_key)
-      state = schedule_broadcast_events(state, [{:join, key, pid, value}])
-      {:reply, :ok, state}
+      {:reply, {:error, :already_present}, state}
     end
   end
 
-  def handle_call({:leave, key, pid}, _from, state) do
-    %{values: values, pids: pids} = state
-    ets_key = {key, pid}
-    ms = [{ets_key, [], [true]}]
-
-    case :ets.select_delete(pids, ms) do
-      0 ->
-        {:reply, {:error, :not_member}, state}
-
-      1 ->
-        unless :ets.member(pids, pid) do
-          Process.unlink(pid)
+  def handle_call({:update, key, pid, arg}, _from, state) do
+    %{store: store} = state
+
+    case Store.fetch(store, key, pid) do
+      {:ok, value, delta} ->
+        case local_update(arg, key, pid, delta, value, state) do
+          {:put, value, delta, state} ->
+            store = Store.local_update(store, key, pid, value, delta)
+            {:reply, :ok, %{state | store: store}}
+
+          {:delete, value, _delta} ->
+            # TODO: this delta has to propagate remotely before delete
+            case Store.local_delete(store, key, pid) do
+              # The value returned from update is fresher
+              {:ok, _value, store} ->
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                state = local_delete([value], state)
+                {:reply, :ok, %{state | store: store}}
+
+              {:last_member, _value, store} ->
+                unlink_flush(pid)
+                state = local_delete([value], state)
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                {:reply, :ok, %{state | store: store}}
+            end
         end
 
-        :ets.delete(values, ets_key)
-        state = schedule_broadcast_events(state, [{:leave, key, pid}])
-        {:reply, :ok, state}
+      :error ->
+        {:reply, {:error, :not_present}, state}
     end
   end
 
-  def handle_call({:update, key, pid, update}, _from, state) do
-    %{values: values} = state
-    ets_key = {key, pid}
-
-    case ets_fetch_element(values, ets_key, 2) do
-      {:ok, value} ->
-        new_value = update.(value)
-        :ets.insert(values, {ets_key, new_value})
-        state = schedule_broadcast_events(state, [{:replace, key, pid, new_value}])
-        {:reply, :ok, state}
+  def handle_call({:delete, key, pid}, _from, state) do
+    %{store: store} = state
 
-      :error ->
-        {:reply, {:error, :not_member}, state}
-    end
-  end
+    case Store.local_delete(store, key, pid) do
+      {:ok, value, store} ->
+        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+        state = local_delete([value], state)
+        {:reply, :ok, %{state | store: store}}
 
-  def handle_call({:replace, key, pid, value}, _from, state) do
-    %{values: values} = state
-    ets_key = {key, pid}
+      {:last_member, value, store} ->
+        unlink_flush(pid)
+        state = local_delete([value], state)
+        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+        {:reply, :ok, %{state | store: store}}
 
-    if :ets.member(values, ets_key) do
-      :ets.insert(values, {ets_key, value})
-      state = schedule_broadcast_events(state, [{:replace, key, pid, value}])
-      {:reply, :ok, state}
-    else
-      {:reply, {:error, :not_member}, state}
+      {:error, store} ->
+        {:reply, {:error, :not_present}, %{state | store: store}}
     end
   end
 
-  def handle_call({:leave, pid}, _from, state) do
-    %{values: values, pids: pids} = state
+  def handle_call({:delete, pid}, _from, state) do
+    %{store: store} = state
 
-    case untrack_pid(pids, values, pid) do
-      {:ok, leaves} ->
+    case Store.local_delete(store, pid) do
+      {:ok, leaves, store} ->
         unlink_flush(pid)
-        state = schedule_broadcast_events(state, leaves)
-        {:reply, :ok, state}
+        state = local_delete(leaves, state)
+        # state = schedule_broadcast_events(state, leaves)
+        {:reply, :ok, %{state | store: store}}
 
-      :error ->
-        {:reply, {:error, :not_member}, state}
+      {:error, store} ->
+        {:reply, {:error, :not_member}, %{state | store: store}}
     end
   end
 
-  def handle_call({:members, key}, _from, state) do
-    %{values: values} = state
+  def handle_call({:list, key}, _from, state) do
+    %{store: store} = state
 
-    read = fn ->
-      local = {{{key, :_}, :"$1"}, [], [:"$1"]}
-      remote = {{{key, :_}, :_, :"$1"}, [], [:"$1"]}
-      :ets.select(values, [local, remote])
-    end
+    read = fn -> Store.list(store, key) end
 
     {:reply, read, state}
   end
 
   @impl true
   def handle_info({:EXIT, pid, reason}, state) do
-    %{values: values, pids: pids} = state
+    %{store: store} = state
 
-    case untrack_pid(pids, values, pid) do
-      {:ok, leaves} ->
-        state = schedule_broadcast_events(state, leaves)
-        {:noreply, state}
+    case Store.local_delete(store, pid) do
+      {:ok, leaves, store} ->
+        state = local_delete(leaves, state)
+        # state = schedule_broadcast_events(state, leaves)
+        {:noreply, %{state | store: store}}
 
-      :error ->
-        {:stop, reason, state}
+      {:error, store} ->
+        {:stop, reason, %{state | store: store}}
     end
   end
 
@@ -473,6 +486,39 @@ defmodule Firenest.ReplicatedState.Server do
     {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
   end
 
+  def handle_info({:update, key, pid, arg}, state) do
+    %{store: store} = state
+
+    case Store.fetch(store, key, pid) do
+      {:ok, value, delta} ->
+        case local_update(arg, key, pid, delta, value, state) do
+          {:put, value, delta, state} ->
+            store = Store.local_update(store, key, pid, value, delta)
+            {:noreply, %{state | store: store}}
+
+          {:delete, value, _delta} ->
+            # TODO: this delta has to propagate remotely before delete
+            case Store.local_delete(store, key, pid) do
+              # The value returned from update is fresher
+              {:ok, _value, store} ->
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                state = local_delete([value], state)
+                {:noreply, %{state | store: store}}
+
+              {:last_member, _value, store} ->
+                unlink_flush(pid)
+                state = local_delete([value], state)
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                {:noreply, %{state | store: store}}
+            end
+        end
+
+      :error ->
+        # Must have been already deleted, ignore
+        {:noreply, state}
+    end
+  end
+
   @impl true
   def handle_remote({:catch_up_req, clock}, from, state) do
     {mode, data} = catch_up_reply(state, clock)
@@ -532,32 +578,57 @@ defmodule Firenest.ReplicatedState.Server do
     {:noreply, %{state | remote_clocks: Map.delete(remote_clocks, remote_ref)}}
   end
 
-  defp untrack_pid(pids, values, pid) do
-    case :ets.take(pids, pid) do
-      [] ->
-        :error
+  defp local_put(arg, key, pid, state) do
+    %{handler: handler, config: config} = state
+
+    case handler.local_put(arg, config) do
+      {delta, value} ->
+        {:put, value, delta, state}
 
-      list ->
-        ms = for ets_key <- list, do: {{ets_key, :_}, [], [true]}
-        :ets.select_delete(values, ms)
-        leaves = for {key, pid} <- list, do: {:leave, key, pid}
-        {:ok, leaves}
+      {delta, value, :delete} ->
+        {:delete, value, delta}
+
+      {delta, value, {:update_after, update, time}} ->
+        Process.send_after(self(), {:update, key, pid, update}, time)
+        {:put, value, delta, state}
     end
   end
 
-  defp ets_fetch_element(table, key, pos) do
-    {:ok, :ets.lookup_element(table, key, pos)}
-  catch
-    :error, :badarg -> :error
+  defp local_update(arg, key, pid, local_delta, value, state) do
+    %{handler: handler, config: config} = state
+
+    case handler.local_update(arg, local_delta, value, config) do
+      {delta, value} ->
+        {:put, value, delta, state}
+
+      {delta, value, :delete} ->
+        {:delete, value, delta}
+
+      {delta, value, {:update_after, update, time}} ->
+        Process.send_after(self(), {:update, key, pid, update}, time)
+        {:put, value, delta, state}
+    end
+  end
+
+  defp local_delete(leaves, state) do
+    %{handler: handler, config: config} = state
+
+    Enum.each(leaves, &handler.local_delete(&1, config))
+    state
   end
 
+  defp link(:partition), do: true
+  defp link(pid), do: Process.link(pid)
+
+  defp unlink_flush(:partition), do: true
+
   defp unlink_flush(pid) do
     Process.unlink(pid)
 
     receive do
-      {:EXIT, ^pid, _} -> :ok
+      {:EXIT, ^pid, _} -> true
     after
-      0 -> :ok
+      0 -> true
     end
   end
 
@@ -615,10 +686,4 @@ defmodule Firenest.ReplicatedState.Server do
     local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
     {:state_transfer, {clock, :ets.select(values, local_ms)}}
   end
-
-  if function_exported?(:ets, :whereis, 1) do
-    defp ets_whereis(table), do: :ets.whereis(table)
-  else
-    defp ets_whereis(table), do: table
-  end
 end
diff --git a/lib/firenest/replicated_state/store.ex b/lib/firenest/replicated_state/store.ex
new file mode 100644
index 0000000..aa7616d
--- /dev/null
+++ b/lib/firenest/replicated_state/store.ex
@@ -0,0 +1,84 @@
+defmodule Firenest.ReplicatedState.Store do
+  defstruct [:values, :pids]
+
+  def new(name) do
+    values = :ets.new(name, [:named_table, :protected, :ordered_set])
+    pids = :ets.new(__MODULE__.Pids, [:duplicate_bag, keypos: 2])
+
+    %__MODULE__{values: ets_whereis(values), pids: pids}
+  end
+
+  def list(%__MODULE__{values: values}, key) do
+    local = {{{key, :_}, :"$1", :_}, [], [:"$1"]}
+    # remote = {{{key, :_}, :_, :"$1"}, [], [:"$1"]}
+    :ets.select(values, [local])
+  end
+
+  def present?(%__MODULE__{values: values}, key, pid) do
+    ets_key = {key, pid}
+    :ets.member(values, ets_key)
+  end
+
+  def fetch(%__MODULE__{values: values}, key, pid) do
+    ets_key = {key, pid}
+
+    case :ets.match(values, {ets_key, :"$1", :"$2"}) do
+      [[value, local_delta]] -> {:ok, value, local_delta}
+      [] -> :error
+    end
+  end
+
+  def local_put(%__MODULE__{values: values, pids: pids} = state, key, pid, value, local_delta) do
+    ets_key = {key, pid}
+
+    true = :ets.insert_new(values, {ets_key, value, local_delta})
+    :ets.insert(pids, ets_key)
+    state
+  end
+
+  def local_delete(%__MODULE__{values: values, pids: pids} = state, key, pid) do
+    ets_key = {key, pid}
+    ms = [{ets_key, [], [true]}]
+
+    case :ets.select_delete(pids, ms) do
+      0 ->
+        {:error, state}
+
+      1 ->
+        [{_, value, _}] = :ets.take(values, ets_key)
+
+        if :ets.member(pids, pid) do
+          {:ok, value, state}
+        else
+          {:last_member, value, state}
+        end
+    end
+  end
+
+  def local_delete(%__MODULE__{values: values, pids: pids} = state, pid) do
+    case :ets.take(pids, pid) do
+      [] ->
+        {:error, state}
+
+      list ->
+        delete_ms = for ets_key <- list, do: {{ets_key, :_, :_}, [], [true]}
+        select_ms = for ets_key <- list, do: {{ets_key, :"$1", :_}, [], [:"$1"]}
+        data = :ets.select(values, select_ms)
+        :ets.select_delete(values, delete_ms)
+        {:ok, data, state}
+    end
+  end
+
+  def local_update(%__MODULE__{values: values} = state, key, pid, value, local_delta) do
+    ets_key = {key, pid}
+
+    :ets.insert(values, {ets_key, value, local_delta})
+    state
+  end
+
+  if function_exported?(:ets, :whereis, 1) do
+    defp ets_whereis(table), do: :ets.whereis(table)
+  else
+    defp ets_whereis(table), do: table
+  end
+end
diff --git a/test/firenest/pub_sub_test.exs b/test/firenest/pub_sub_test.exs
index da9ff8c..9a5ebf6 100644
--- a/test/firenest/pub_sub_test.exs
+++ b/test/firenest/pub_sub_test.exs
@@ -7,7 +7,7 @@ defmodule Firenest.PubSubTest do
   import Firenest.TestHelpers
 
   setup_all do
-    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
+    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 500)
     nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
     pubsub = Firenest.Test.PubSub
     topology = Firenest.Test
diff --git a/test/firenest/replicated_state_test.exs b/test/firenest/replicated_state_test.exs
index 355c4dc..2ef1b23 100644
--- a/test/firenest/replicated_state_test.exs
+++ b/test/firenest/replicated_state_test.exs
@@ -11,28 +11,38 @@ defmodule Firenest.ReplicatedStateTest do
   end
 
   setup %{test: test, topology: topology} do
-    assert {:ok, _} = start_supervised({R, name: test, topology: topology})
+    opts = [name: test, topology: topology, handler: Firenest.Test.EvalState]
+    assert {:ok, _} = start_supervised({R, opts})
     {:ok, server: test}
   end
 
-  describe "join/5" do
+  describe "put/4" do
     test "adds process", %{server: server} do
-      assert R.join(server, :foo, self(), :baz) == :ok
-      assert [:baz] == R.members(server, :foo)
+      parent = self()
+
+      fun = fn config ->
+        send(parent, {:local_put, config})
+        {:ok, 1}
+      end
+
+      assert R.put(server, :foo, self(), fun) == :ok
+      assert_received {:local_put, _}
+
+      assert [1] == R.list(server, :foo)
     end
 
-    test "rejects double joins", %{server: server} do
-      assert R.join(server, :foo, self(), :baz) == :ok
-      assert R.join(server, :foo, self(), :baz) == {:error, :already_joined}
+    test "rejects double puts", %{server: server} do
+      assert R.put(server, :foo, self(), 1) == :ok
+      assert R.put(server, :foo, self(), 2) == {:error, :already_present}
     end
 
     test "cleans up entries after process dies", %{server: server} do
       {pid, ref} = spawn_monitor(Process, :sleep, [:infinity])
-      R.join(server, :foo, pid, :baz)
-      assert [_] = R.members(server, :foo)
+      R.put(server, :foo, pid, 1)
+      assert [_] = R.list(server, :foo)
       Process.exit(pid, :kill)
       assert_receive {:DOWN, ^ref, _, _, _}
-      assert [] = R.members(server, :foo)
+      assert [] = R.list(server, :foo)
     end
 
     test "server dies if other linked process dies", %{server: server} do
@@ -52,177 +62,277 @@ defmodule Firenest.ReplicatedStateTest do
       Process.exit(temp, :shutdown)
       assert_receive {:DOWN, ^ref, _, _, _}
     end
+
+    test "immediately deletes with :delete return", %{server: server} do
+      parent = self()
+
+      fun = fn config ->
+        send(parent, {:local_put, config})
+
+        delete = fn config ->
+          send(parent, {:local_delete, config})
+        end
+
+        {:ok, delete, :delete}
+      end
+
+      assert R.put(server, :foo, self(), fun) == :ok
+      assert_received {:local_put, _}
+      assert_received {:local_delete, _}
+
+      assert [] == R.list(server, :foo)
+    end
+
+    test "updates on a timeout with :update_after return", %{server: server} do
+      parent = self()
+
+      fun = fn config ->
+        send(parent, {:local_put, config})
+
+        update = fn delta, state, config ->
+          send(parent, {:local_update, delta, state, config})
+          {delta + 1, state + 1}
+        end
+
+        {1, 1, {:update_after, update, 50}}
+      end
+
+      assert R.put(server, :foo, self(), fun) == :ok
+      assert_received {:local_put, _}
+      refute_received {:local_update, _, _, _}
+      assert [1] == R.list(server, :foo)
+
+      assert_receive {:local_update, 1, 1, _}
+      assert [2] == R.list(server, :foo)
+    end
   end
 
-  describe "leave/2" do
+  describe "delete/2" do
     test "removes entry", %{server: server} do
-      R.join(server, :foo, self(), :baz)
+      parent = self()
 
-      assert [_] = R.members(server, :foo)
-      assert R.leave(server, self()) == :ok
-      assert [] == R.members(server, :foo)
+      fun = fn config ->
+        send(parent, {:local_delete, config})
+      end
+
+      R.put(server, :foo, self(), fn _ -> {:ok, fun} end)
+
+      assert [_] = R.list(server, :foo)
+      assert R.delete(server, self()) == :ok
+      assert_received {:local_delete, _}
+
+      assert [] == R.list(server, :foo)
     end
 
     test "does not remove non members", %{server: server} do
       [{_, pid, _, _}] = Supervisor.which_children(Module.concat(server, "Supervisor"))
       Process.link(pid)
 
-      assert R.leave(server, self()) == {:error, :not_member}
+      assert R.delete(server, self()) == {:error, :not_member}
       {:links, links} = Process.info(self(), :links)
       assert pid in links
     end
   end
 
-  describe "leave/4" do
+  describe "delete/3" do
     test "removes single entry", %{server: server} do
-      R.join(server, :foo, self(), :baz)
-      assert [_] = R.members(server, :foo)
+      parent = self()
+
+      fun = fn config ->
+        send(parent, {:local_delete, config})
+      end
 
-      assert R.leave(server, :foo, self()) == :ok
-      assert [] == R.members(server, :foo)
+      R.put(server, :foo, self(), fn _ -> {:ok, fun} end)
+      assert [_] = R.list(server, :foo)
+
+      assert R.delete(server, :foo, self()) == :ok
+      assert_received {:local_delete, _}
+      assert [] == R.list(server, :foo)
     end
 
     test "leaves other entries intact", %{server: server} do
       pid = spawn_link(fn -> Process.sleep(:infinity) end)
-      R.join(server, :foo, self(), :baz)
-      R.join(server, :foo, pid, :baaz)
-      assert [_, _] = R.members(server, :foo)
+      R.put(server, :foo, self(), 1)
+      R.put(server, :foo, pid, 2)
+      assert [_, _] = R.list(server, :foo)
 
-      assert R.leave(server, :foo, self()) == :ok
-      assert [:baaz] == R.members(server, :foo)
+      assert R.delete(server, :foo, self()) == :ok
+      assert [2] == R.list(server, :foo)
     end
 
     test "does not remove non members", %{server: server} do
       [{_, pid, _, _}] = Supervisor.which_children(Module.concat(server, "Supervisor"))
       Process.link(pid)
 
-      assert R.leave(server, :foo, self()) == {:error, :not_member}
+      assert R.delete(server, :foo, self()) == {:error, :not_present}
       {:links, links} = Process.info(self(), :links)
       assert pid in links
     end
   end
 
-  describe "update/5" do
+  describe "update/4" do
     test "executes the update if entry is present", %{server: server} do
       parent = self()
-      R.join(server, :foo, self(), 1)
-      assert [1] == R.members(server, :foo)
 
-      update = fn value ->
-        send(parent, value)
-        value + 1
+      R.put(server, :foo, self(), 1)
+      assert [1] = R.list(server, :foo)
+
+      fun = fn delta, state, config ->
+        send(parent, {:local_update, delta, state, config})
+        {delta + 1, state + 1}
       end
 
-      assert R.update(server, :foo, self(), update) == :ok
-      assert_received 1
-      assert [2] == R.members(server, :foo)
+      assert R.update(server, :foo, self(), fun) == :ok
+      assert_received {:local_update, 1, 1, _}
+      assert [2] = R.list(server, :foo)
     end
 
     test "does not execute update if entry is absent", %{server: server} do
       parent = self()
-      update = fn value -> Process.exit(parent, {:unexpected_update, value}) end
-      assert R.update(server, :foo, self(), update) == {:error, :not_member}
-    end
-  end
-
-  describe "replace/5" do
-    test "updates value if entry is present", %{server: server} do
-      R.join(server, :foo, self(), 1)
-      assert [1] == R.members(server, :foo)
-      assert R.replace(server, :foo, self(), 2) == :ok
-      assert [2] == R.members(server, :foo)
-    end
-
-    test "does not update value if entry is absent", %{server: server} do
-      assert R.replace(server, :foo, self(), 2) == {:error, :not_member}
-    end
-  end
-
-  defmodule Distributed do
-    # We modify test topology, it can't be async
-    use ExUnit.Case
-
-    setup_all do
-      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
-      nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
-      topology = Firenest.Test
-      server = Firenest.Test.ReplicatedServer
-      %{start: start} = R.child_spec(name: server, topology: topology)
-      Firenest.Test.start_link(nodes, start)
-      nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
-
-      {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, server: server}
-    end
-
-    test "remote join is propagated", config do
-      %{server: server, test: test, nodes: [second | _]} = config
 
-      quote do
-        spawn(fn ->
-          :ok = R.join(unquote(server), unquote(test), self(), :baz)
-          :timer.sleep(:infinity)
-        end)
+      fun = fn delta, state, config ->
+        Process.exit(parent, {:unexpected_update, delta, state, config})
+        {delta, state}
       end
-      |> eval_on_node(second, config)
 
-      wait_until(fn -> R.members(server, test) == [:baz] end)
+      assert R.update(server, :foo, self(), fun) == {:error, :not_present}
     end
 
-    test "propages changes when nodes were disconnected", config do
-      %{topology: topology, server: server, test: test, nodes: [second, third]} = config
-      Process.register(self(), test)
+    test "immediately deletes with :delete return", %{server: server} do
+      parent = self()
 
-      quote do
-        spawn(fn ->
-          Process.register(self(), unquote(test))
-          :ok = R.join(unquote(server), unquote(test), self(), :baz)
-          Process.sleep(:infinity)
-        end)
-      end
-      |> eval_on_node(second, config)
+      R.put(server, :foo, self(), 1)
+      assert [1] = R.list(server, :foo)
 
-      quote(do: R.members(unquote(server), unquote(test)) == [:baz])
-      |> await_on_node(third, config)
+      fun = fn delta, state, config ->
+        send(parent, {:local_update, delta, state, config})
 
-      quote do
-        T.disconnect(unquote(topology), elem(unquote(third), 0))
-        pid = Process.whereis(unquote(test))
-        :ok = R.leave(unquote(server), unquote(test), pid)
+        delete = fn config ->
+          send(parent, {:local_delete, config})
+        end
 
-        spawn(fn ->
-          :ok = R.join(unquote(server), unquote(test), self(), :bar)
-          Process.sleep(:infinity)
-        end)
+        {delta + 1, delete, :delete}
       end
-      |> eval_on_node(second, config)
 
-      quote(do: R.members(unquote(server), unquote(test)) == [])
-      |> await_on_node(third, config)
+      assert R.update(server, :foo, self(), fun) == :ok
+      assert_received {:local_update, 1, 1, _}
+      assert_received {:local_delete, _}
 
-      quote(do: T.connect(unquote(topology), elem(unquote(third), 0)))
-      |> eval_on_node(second, config)
-
-      quote(do: R.members(unquote(server), unquote(test)) == [:bar])
-      |> await_on_node(third, config)
+      assert [] == R.list(server, :foo)
     end
 
-    defp eval_on_node(quoted, node, config) do
-      %{topology: topology, evaluator: evaluator} = config
+    test "updates on a timeout with :update_after return", %{server: server} do
+      parent = self()
+
+      R.put(server, :foo, self(), 1)
+      assert [1] = R.list(server, :foo)
 
-      T.send(topology, node, evaluator, {:eval_quoted, quoted})
-    end
+      fun = fn delta, state, config ->
+        send(parent, {:local_update, delta, state, config})
 
-    defp await_on_node(quoted, node, config) do
-      %{topology: topology} = config
-      {:registered_name, name} = Process.info(self(), :registered_name)
+        update = fn delta, state, config ->
+          send(parent, {:local_update, delta, state, config})
+          {delta + 1, state + 1}
+        end
 
-      quote do
-        wait_until(fn -> unquote(quoted) end)
-        T.broadcast(unquote(topology), unquote(name), :continue)
+        {delta + 1, state + 1, {:update_after, update, 50}}
       end
-      |> eval_on_node(node, config)
 
-      assert_receive :continue
+      assert R.update(server, :foo, self(), fun) == :ok
+      assert_received {:local_update, 1, 1, _}
+      refute_received {:local_update, _, _, _}
+      assert [2] == R.list(server, :foo)
+
+      assert_receive {:local_update, 2, 2, _}
+      assert [3] == R.list(server, :foo)
     end
   end
+
+  # defmodule Distributed do
+  #   # We modify test topology, it can't be async
+  #   use ExUnit.Case
+
+  #   setup_all do
+  #     wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
+  #     nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
+  #     topology = Firenest.Test
+  #     server = Firenest.Test.ReplicatedServer
+  #     %{start: start} = R.child_spec(name: server, topology: topology)
+  #     Firenest.Test.start_link(nodes, start)
+  #     nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
+
+  #     {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, server: server}
+  #   end
+
+  #   test "remote join is propagated", config do
+  #     %{server: server, test: test, nodes: [second | _]} = config
+
+  #     quote do
+  #       spawn(fn ->
+  #         :ok = R.join(unquote(server), unquote(test), self(), :baz)
+  #         :timer.sleep(:infinity)
+  #       end)
+  #     end
+  #     |> eval_on_node(second, config)
+
+  #     wait_until(fn -> R.members(server, test) == [:baz] end)
+  #   end
+
+  #   test "propages changes when nodes were disconnected", config do
+  #     %{topology: topology, server: server, test: test, nodes: [second, third]} = config
+  #     Process.register(self(), test)
+
+  #     quote do
+  #       spawn(fn ->
+  #         Process.register(self(), unquote(test))
+  #         :ok = R.join(unquote(server), unquote(test), self(), :baz)
+  #         Process.sleep(:infinity)
+  #       end)
+  #     end
+  #     |> eval_on_node(second, config)
+
+  #     quote(do: R.members(unquote(server), unquote(test)) == [:baz])
+  #     |> await_on_node(third, config)
+
+  #     quote do
+  #       T.disconnect(unquote(topology), elem(unquote(third), 0))
+  #       pid = Process.whereis(unquote(test))
+  #       :ok = R.leave(unquote(server), unquote(test), pid)
+
+  #       spawn(fn ->
+  #         :ok = R.join(unquote(server), unquote(test), self(), :bar)
+  #         Process.sleep(:infinity)
+  #       end)
+  #     end
+  #     |> eval_on_node(second, config)
+
+  #     quote(do: R.members(unquote(server), unquote(test)) == [])
+  #     |> await_on_node(third, config)
+
+  #     quote(do: T.connect(unquote(topology), elem(unquote(third), 0)))
+  #     |> eval_on_node(second, config)
+
+  #     quote(do: R.members(unquote(server), unquote(test)) == [:bar])
+  #     |> await_on_node(third, config)
+  #   end
+
+  #   defp eval_on_node(quoted, node, config) do
+  #     %{topology: topology, evaluator: evaluator} = config
+
+  #     T.send(topology, node, evaluator, {:eval_quoted, quoted})
+  #   end
+
+  #   defp await_on_node(quoted, node, config) do
+  #     %{topology: topology} = config
+  #     {:registered_name, name} = Process.info(self(), :registered_name)
+
+  #     quote do
+  #       wait_until(fn -> unquote(quoted) end)
+  #       T.broadcast(unquote(topology), unquote(name), :continue)
+  #     end
+  #     |> eval_on_node(node, config)
+
+  #     assert_receive :continue
+  #   end
+  # end
 end
diff --git a/test/firenest/synced_server_test.exs b/test/firenest/synced_server_test.exs
index 5595fd7..a7fd032 100644
--- a/test/firenest/synced_server_test.exs
+++ b/test/firenest/synced_server_test.exs
@@ -323,7 +323,7 @@ defmodule Firenest.SyncedServerTest do
     use ExUnit.Case, async: true
 
     setup_all do
-      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
+      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 500)
       nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
       topology = Firenest.Test
       nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
diff --git a/test/support/eval_state.ex b/test/support/eval_state.ex
new file mode 100644
index 0000000..aa68323
--- /dev/null
+++ b/test/support/eval_state.ex
@@ -0,0 +1,20 @@
+defmodule Firenest.Test.EvalState do
+  @behaviour Firenest.ReplicatedState
+
+  @impl true
+  def init(opts) do
+    {0, opts}
+  end
+
+  @impl true
+  def local_put(fun, config) when is_function(fun, 1), do: fun.(config)
+  def local_put(data, _config), do: {data, data}
+
+  @impl true
+  def local_delete(fun, config) when is_function(fun, 1), do: fun.(config)
+  def local_delete(_data, _config), do: :ok
+
+  @impl true
+  def local_update(fun, delta, state, config) when is_function(fun, 3),
+    do: fun.(delta, state, config)
+end

From 94aa7cbf8bec8abe1e403b25194b76b4b0b4da89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 8 Aug 2018 14:59:35 +0200
Subject: [PATCH 20/40] Initial delta returned again in init/1

---
 lib/firenest/replicated_state.ex        | 28 ++++++++++++-------
 test/firenest/replicated_state_test.exs | 36 ++++++++++++-------------
 test/support/eval_state.ex              |  8 +++---
 3 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 10b54de..0d3d4d2 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -37,12 +37,12 @@ defmodule Firenest.ReplicatedState do
   @doc """
   Called when a partition starts up.
 
-  It returns an immutable `config` value that will be passed to
+  It returns an `initial_delta` that will be passed to `c:local_put/3`
+  callback on new entries and to `c:local_update/4` after remote
+  broadcast resets the local delta.
+  It also returns an immutable `config` value that will be passed to
   all callbacks.
   """
-  # TODO: initial delta is returned here and passed to local_put.
-  # The reason: when we reset the local delta after broadcast, we'd
-  # need to store the initial_delta for each data point separately
   @callback init(opts :: keyword()) :: {initial_delta :: local_delta(), config()}
 
   @doc """
@@ -52,17 +52,24 @@ defmodule Firenest.ReplicatedState do
   explanation of the `extra_action` return values see the
   `c:local_update/4` callback.
 
+  The value of `local_delta` argument is always the initial delta as
+  returned by the `init/1` callback.
+
   This is a good place for broadcasting local state changes.
   """
-  @callback local_put(arg :: term(), config()) ::
-              {initial_delta, initial_state} | {initial_delta, initial_state, extra_action()}
-            when initial_delta: local_delta(), initial_state: state()
+  @callback local_put(arg :: term(), local_delta(), config()) ::
+              {local_delta(), initial_state} | {local_delta(), initial_state, extra_action()}
+            when initial_state: state()
 
   @doc """
   Called whenever the `update/4` function is called to update a state.
 
   This is a good place for broadcasting local state changes.
 
+  The `local_delta` argument is either the accumulated delta from
+  `c:local_put/3` and `c:local_update/4` calls or the initial delta
+  from `c:init/1` after remote broadcast resets the local delta.
+
   ## Delayed update and delete
 
   If the function returns a third element in the tuple consisting of
@@ -349,12 +356,13 @@ defmodule Firenest.ReplicatedState.Server do
     store = Store.new(name)
     broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
 
-    config = handler.init(opts)
+    {initial_delta, config} = handler.init(opts)
 
     {:ok,
      %{
        store: store,
        config: config,
+       initial_delta: initial_delta,
        handler: handler,
        broadcast_timer: nil,
        broadcast_timeout: broadcast_timeout,
@@ -579,9 +587,9 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   defp local_put(arg, key, pid, state) do
-    %{handler: handler, config: config} = state
+    %{handler: handler, config: config, initial_delta: delta} = state
 
-    case handler.local_put(arg, config) do
+    case handler.local_put(arg, delta, config) do
       {delta, value} ->
         {:put, value, delta, state}
 
diff --git a/test/firenest/replicated_state_test.exs b/test/firenest/replicated_state_test.exs
index 2ef1b23..69f5129 100644
--- a/test/firenest/replicated_state_test.exs
+++ b/test/firenest/replicated_state_test.exs
@@ -20,13 +20,13 @@ defmodule Firenest.ReplicatedStateTest do
     test "adds process", %{server: server} do
       parent = self()
 
-      fun = fn config ->
-        send(parent, {:local_put, config})
-        {:ok, 1}
+      fun = fn delta, config ->
+        send(parent, {:local_put, delta, config})
+        {delta + 1, 1}
       end
 
       assert R.put(server, :foo, self(), fun) == :ok
-      assert_received {:local_put, _}
+      assert_received {:local_put, 0, _}
 
       assert [1] == R.list(server, :foo)
     end
@@ -66,18 +66,18 @@ defmodule Firenest.ReplicatedStateTest do
     test "immediately deletes with :delete return", %{server: server} do
       parent = self()
 
-      fun = fn config ->
-        send(parent, {:local_put, config})
+      fun = fn delta, config ->
+        send(parent, {:local_put, delta, config})
 
         delete = fn config ->
           send(parent, {:local_delete, config})
         end
 
-        {:ok, delete, :delete}
+        {delta + 1, delete, :delete}
       end
 
       assert R.put(server, :foo, self(), fun) == :ok
-      assert_received {:local_put, _}
+      assert_received {:local_put, 0, _}
       assert_received {:local_delete, _}
 
       assert [] == R.list(server, :foo)
@@ -86,19 +86,19 @@ defmodule Firenest.ReplicatedStateTest do
     test "updates on a timeout with :update_after return", %{server: server} do
       parent = self()
 
-      fun = fn config ->
-        send(parent, {:local_put, config})
+      fun = fn delta, config ->
+        send(parent, {:local_put, delta, config})
 
         update = fn delta, state, config ->
           send(parent, {:local_update, delta, state, config})
           {delta + 1, state + 1}
         end
 
-        {1, 1, {:update_after, update, 50}}
+        {delta + 1, 1, {:update_after, update, 50}}
       end
 
       assert R.put(server, :foo, self(), fun) == :ok
-      assert_received {:local_put, _}
+      assert_received {:local_put, 0, _}
       refute_received {:local_update, _, _, _}
       assert [1] == R.list(server, :foo)
 
@@ -115,7 +115,7 @@ defmodule Firenest.ReplicatedStateTest do
         send(parent, {:local_delete, config})
       end
 
-      R.put(server, :foo, self(), fn _ -> {:ok, fun} end)
+      R.put(server, :foo, self(), fn delta, _ -> {delta, fun} end)
 
       assert [_] = R.list(server, :foo)
       assert R.delete(server, self()) == :ok
@@ -142,7 +142,7 @@ defmodule Firenest.ReplicatedStateTest do
         send(parent, {:local_delete, config})
       end
 
-      R.put(server, :foo, self(), fn _ -> {:ok, fun} end)
+      R.put(server, :foo, self(), fn delta, _ -> {delta, fun} end)
       assert [_] = R.list(server, :foo)
 
       assert R.delete(server, :foo, self()) == :ok
@@ -183,7 +183,7 @@ defmodule Firenest.ReplicatedStateTest do
       end
 
       assert R.update(server, :foo, self(), fun) == :ok
-      assert_received {:local_update, 1, 1, _}
+      assert_received {:local_update, 0, 1, _}
       assert [2] = R.list(server, :foo)
     end
 
@@ -215,7 +215,7 @@ defmodule Firenest.ReplicatedStateTest do
       end
 
       assert R.update(server, :foo, self(), fun) == :ok
-      assert_received {:local_update, 1, 1, _}
+      assert_received {:local_update, 0, 1, _}
       assert_received {:local_delete, _}
 
       assert [] == R.list(server, :foo)
@@ -239,11 +239,11 @@ defmodule Firenest.ReplicatedStateTest do
       end
 
       assert R.update(server, :foo, self(), fun) == :ok
-      assert_received {:local_update, 1, 1, _}
+      assert_received {:local_update, 0, 1, _}
       refute_received {:local_update, _, _, _}
       assert [2] == R.list(server, :foo)
 
-      assert_receive {:local_update, 2, 2, _}
+      assert_receive {:local_update, 1, 2, _}
       assert [3] == R.list(server, :foo)
     end
   end
diff --git a/test/support/eval_state.ex b/test/support/eval_state.ex
index aa68323..b49fe26 100644
--- a/test/support/eval_state.ex
+++ b/test/support/eval_state.ex
@@ -7,14 +7,14 @@ defmodule Firenest.Test.EvalState do
   end
 
   @impl true
-  def local_put(fun, config) when is_function(fun, 1), do: fun.(config)
-  def local_put(data, _config), do: {data, data}
+  def local_put(fun, delta, config) when is_function(fun), do: fun.(delta, config)
+  def local_put(data, delta, _config), do: {delta, data}
 
   @impl true
-  def local_delete(fun, config) when is_function(fun, 1), do: fun.(config)
+  def local_delete(fun, config) when is_function(fun), do: fun.(config)
   def local_delete(_data, _config), do: :ok
 
   @impl true
-  def local_update(fun, delta, state, config) when is_function(fun, 3),
+  def local_update(fun, delta, state, config) when is_function(fun),
     do: fun.(delta, state, config)
 end

From cb87326004df201e96893c31bec3ade60ac7731e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 8 Aug 2018 15:05:24 +0200
Subject: [PATCH 21/40] Move ReplicatedState.Server to its own file

---
 lib/firenest/replicated_state.ex        | 362 ------------------------
 lib/firenest/replicated_state/server.ex | 361 +++++++++++++++++++++++
 2 files changed, 361 insertions(+), 362 deletions(-)
 create mode 100644 lib/firenest/replicated_state/server.ex

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 0d3d4d2..a95ee71 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -333,365 +333,3 @@ defmodule Firenest.ReplicatedState.Supervisor do
     Supervisor.init(children, strategy: :one_for_one)
   end
 end
-
-defmodule Firenest.ReplicatedState.Server do
-  @moduledoc false
-  use Firenest.SyncedServer
-
-  alias Firenest.SyncedServer
-  alias Firenest.ReplicatedState.Store
-
-  def child_spec({name, topology, handler, opts}) do
-    server_opts = [name: name, topology: topology]
-
-    %{
-      id: name,
-      start: {SyncedServer, :start_link, [__MODULE__, {name, handler, opts}, server_opts]}
-    }
-  end
-
-  @impl true
-  def init({name, handler, opts}) do
-    Process.flag(:trap_exit, true)
-    store = Store.new(name)
-    broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
-
-    {initial_delta, config} = handler.init(opts)
-
-    {:ok,
-     %{
-       store: store,
-       config: config,
-       initial_delta: initial_delta,
-       handler: handler,
-       broadcast_timer: nil,
-       broadcast_timeout: broadcast_timeout,
-       clock: 0,
-       remote_clocks: %{},
-       pending_events: []
-     }}
-  end
-
-  @impl true
-  def handshake_data(%{clock: clock}), do: clock
-
-  @impl true
-  def handle_call({:put, key, pid, arg}, _from, state) do
-    %{store: store} = state
-
-    link(pid)
-
-    unless Store.present?(store, key, pid) do
-      case local_put(arg, key, pid, state) do
-        {:put, value, delta, state} ->
-          store = Store.local_put(store, key, pid, value, delta)
-          {:reply, :ok, %{state | store: store}}
-
-        {:delete, value, _delta} ->
-          # TODO: this delta has to propagate remotely before delete
-          state = local_delete([value], state)
-          {:reply, :ok, state}
-      end
-    else
-      {:reply, {:error, :already_present}, state}
-    end
-  end
-
-  def handle_call({:update, key, pid, arg}, _from, state) do
-    %{store: store} = state
-
-    case Store.fetch(store, key, pid) do
-      {:ok, value, delta} ->
-        case local_update(arg, key, pid, delta, value, state) do
-          {:put, value, delta, state} ->
-            store = Store.local_update(store, key, pid, value, delta)
-            {:reply, :ok, %{state | store: store}}
-
-          {:delete, value, _delta} ->
-            # TODO: this delta has to propagate remotely before delete
-            case Store.local_delete(store, key, pid) do
-              # The value returned from update is fresher
-              {:ok, _value, store} ->
-                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                state = local_delete([value], state)
-                {:reply, :ok, %{state | store: store}}
-
-              {:last_member, _value, store} ->
-                unlink_flush(pid)
-                state = local_delete([value], state)
-                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                {:reply, :ok, %{state | store: store}}
-            end
-        end
-
-      :error ->
-        {:reply, {:error, :not_present}, state}
-    end
-  end
-
-  def handle_call({:delete, key, pid}, _from, state) do
-    %{store: store} = state
-
-    case Store.local_delete(store, key, pid) do
-      {:ok, value, store} ->
-        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-        state = local_delete([value], state)
-        {:reply, :ok, %{state | store: store}}
-
-      {:last_member, value, store} ->
-        unlink_flush(pid)
-        state = local_delete([value], state)
-        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-        {:reply, :ok, %{state | store: store}}
-
-      {:error, store} ->
-        {:reply, {:error, :not_present}, %{state | store: store}}
-    end
-  end
-
-  def handle_call({:delete, pid}, _from, state) do
-    %{store: store} = state
-
-    case Store.local_delete(store, pid) do
-      {:ok, leaves, store} ->
-        unlink_flush(pid)
-        state = local_delete(leaves, state)
-        # state = schedule_broadcast_events(state, leaves)
-        {:reply, :ok, %{state | store: store}}
-
-      {:error, store} ->
-        {:reply, {:error, :not_member}, %{state | store: store}}
-    end
-  end
-
-  def handle_call({:list, key}, _from, state) do
-    %{store: store} = state
-
-    read = fn -> Store.list(store, key) end
-
-    {:reply, read, state}
-  end
-
-  @impl true
-  def handle_info({:EXIT, pid, reason}, state) do
-    %{store: store} = state
-
-    case Store.local_delete(store, pid) do
-      {:ok, leaves, store} ->
-        state = local_delete(leaves, state)
-        # state = schedule_broadcast_events(state, leaves)
-        {:noreply, %{state | store: store}}
-
-      {:error, store} ->
-        {:stop, reason, %{state | store: store}}
-    end
-  end
-
-  def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
-    %{pending_events: events, clock: clock} = state
-    clock = clock + 1
-    SyncedServer.remote_broadcast({:events, clock, events})
-    {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
-  end
-
-  def handle_info({:update, key, pid, arg}, state) do
-    %{store: store} = state
-
-    case Store.fetch(store, key, pid) do
-      {:ok, value, delta} ->
-        case local_update(arg, key, pid, delta, value, state) do
-          {:put, value, delta, state} ->
-            store = Store.local_update(store, key, pid, value, delta)
-            {:noreply, %{state | store: store}}
-
-          {:delete, value, _delta} ->
-            # TODO: this delta has to propagate remotely before delete
-            case Store.local_delete(store, key, pid) do
-              # The value returned from update is fresher
-              {:ok, _value, store} ->
-                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                state = local_delete([value], state)
-                {:noreply, %{state | store: store}}
-
-              {:last_member, _value, store} ->
-                unlink_flush(pid)
-                state = local_delete([value], state)
-                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                {:noreply, %{state | store: store}}
-            end
-        end
-
-      :error ->
-        # Must have been already deleted, ignore
-        {:noreply, state}
-    end
-  end
-
-  @impl true
-  def handle_remote({:catch_up_req, clock}, from, state) do
-    {mode, data} = catch_up_reply(state, clock)
-    SyncedServer.remote_send(from, {:catch_up, mode, data})
-    {:noreply, state}
-  end
-
-  def handle_remote({:catch_up, :state_transfer, {clock, transfer}}, from, state) do
-    state = handle_state_transfer(state, from, clock, transfer)
-    {:noreply, state}
-  end
-
-  def handle_remote({:events, remote_clock, events}, from, state) do
-    %{remote_clocks: remote_clocks} = state
-    local_clock = Map.fetch!(remote_clocks, from)
-
-    if remote_clock == local_clock + 1 do
-      remote_clocks = %{remote_clocks | from => remote_clock}
-      state = handle_events(state, from, events)
-      {:noreply, %{state | remote_clocks: remote_clocks}}
-    else
-      {:noreply, request_catch_up(state, from, local_clock)}
-    end
-  end
-
-  @impl true
-  def handle_replica({:up, remote_clock}, remote_ref, state) do
-    %{remote_clocks: remote_clocks} = state
-
-    case remote_clocks do
-      %{^remote_ref => old_clock} when remote_clock > old_clock ->
-        # Reconnection, try to catch up
-        {:noreply, request_catch_up(state, remote_ref, old_clock)}
-
-      %{^remote_ref => old_clock} ->
-        # Reconnection, no remote state change, skip catch up
-        # Assert for sanity
-        true = old_clock == remote_clock
-        {:noreply, state}
-
-      %{} when remote_clock == 0 ->
-        # New node, no state, don't catch up
-        state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
-        {:noreply, state}
-
-      %{} ->
-        # New node, catch up
-        state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
-        {:noreply, request_catch_up(state, remote_ref, 0)}
-    end
-  end
-
-  def handle_replica(:down, remote_ref, state) do
-    %{values: values, remote_clocks: remote_clocks} = state
-    delete_ms = [{{:_, remote_ref, :_}, [], [true]}]
-    :ets.select_delete(values, delete_ms)
-    {:noreply, %{state | remote_clocks: Map.delete(remote_clocks, remote_ref)}}
-  end
-
-  defp local_put(arg, key, pid, state) do
-    %{handler: handler, config: config, initial_delta: delta} = state
-
-    case handler.local_put(arg, delta, config) do
-      {delta, value} ->
-        {:put, value, delta, state}
-
-      {delta, value, :delete} ->
-        {:delete, value, delta}
-
-      {delta, value, {:update_after, update, time}} ->
-        Process.send_after(self(), {:update, key, pid, update}, time)
-        {:put, value, delta, state}
-    end
-  end
-
-  defp local_update(arg, key, pid, local_delta, value, state) do
-    %{handler: handler, config: config} = state
-
-    case handler.local_update(arg, local_delta, value, config) do
-      {delta, value} ->
-        {:put, value, delta, state}
-
-      {delta, value, :delete} ->
-        {:delete, value, delta}
-
-      {delta, value, {:update_after, update, time}} ->
-        Process.send_after(self(), {:update, key, pid, update}, time)
-        {:put, value, delta, state}
-    end
-  end
-
-  defp local_delete(leaves, state) do
-    %{handler: handler, config: config} = state
-
-    Enum.each(leaves, &handler.local_delete(&1, config))
-    state
-  end
-
-  defp link(:partition), do: true
-  defp link(pid), do: Process.link(pid)
-
-  defp unlink_flush(:partition), do: true
-
-  defp unlink_flush(pid) do
-    Process.unlink(pid)
-
-    receive do
-      {:EXIT, ^pid, _} -> true
-    after
-      0 -> true
-    end
-  end
-
-  defp schedule_broadcast_events(%{broadcast_timer: nil} = state, new_events) do
-    %{broadcast_timeout: timeout, pending_events: events} = state
-    timer = :erlang.start_timer(timeout, self(), :broadcast)
-    %{state | broadcast_timer: timer, pending_events: new_events ++ events}
-  end
-
-  defp schedule_broadcast_events(%{} = state, new_events) do
-    %{pending_events: events} = state
-    %{state | pending_events: new_events ++ events}
-  end
-
-  defp request_catch_up(state, remote_ref, clock) do
-    SyncedServer.remote_send(remote_ref, {:catch_up_req, clock})
-    state
-  end
-
-  defp handle_events(%{values: values} = state, from, events) do
-    {joins, leaves} =
-      Enum.reduce(events, {[], []}, fn
-        {:leave, key, pid}, {joins, leaves} ->
-          leave = {{{key, pid}, from, :_}, [], [true]}
-          {joins, [leave | leaves]}
-
-        {:replace, key, pid, value}, {joins, leaves} ->
-          join = {{key, pid}, from, value}
-          {[join | joins], leaves}
-
-        {:join, key, pid, value}, {joins, leaves} ->
-          join = {{key, pid}, from, value}
-          {[join | joins], leaves}
-      end)
-
-    :ets.insert(values, joins)
-    :ets.select_delete(values, leaves)
-    state
-  end
-
-  # TODO: detect leaves
-  # Is there a better way than to clean up and re-insert?
-  # This can be problematic for dirty reads!
-  defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
-    %{remote_clocks: remote_clocks} = state
-    delete_ms = [{{:_, from, :_}, [], [true]}]
-    inserts = for {ets_key, value} <- transfer, do: {ets_key, from, value}
-    :ets.select_delete(values, delete_ms)
-    :ets.insert(values, inserts)
-    %{state | remote_clocks: %{remote_clocks | from => clock}}
-  end
-
-  # TODO: handle catch-up with events
-  defp catch_up_reply(%{values: values}, clock) do
-    local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
-    {:state_transfer, {clock, :ets.select(values, local_ms)}}
-  end
-end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
new file mode 100644
index 0000000..1be2e2b
--- /dev/null
+++ b/lib/firenest/replicated_state/server.ex
@@ -0,0 +1,361 @@
+defmodule Firenest.ReplicatedState.Server do
+  @moduledoc false
+  use Firenest.SyncedServer
+
+  alias Firenest.SyncedServer
+  alias Firenest.ReplicatedState.Store
+
+  def child_spec({name, topology, handler, opts}) do
+    server_opts = [name: name, topology: topology]
+
+    %{
+      id: name,
+      start: {SyncedServer, :start_link, [__MODULE__, {name, handler, opts}, server_opts]}
+    }
+  end
+
+  @impl true
+  def init({name, handler, opts}) do
+    Process.flag(:trap_exit, true)
+    store = Store.new(name)
+    broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
+
+    {initial_delta, config} = handler.init(opts)
+
+    {:ok,
+     %{
+       store: store,
+       config: config,
+       initial_delta: initial_delta,
+       handler: handler,
+       broadcast_timer: nil,
+       broadcast_timeout: broadcast_timeout,
+       clock: 0,
+       remote_clocks: %{},
+       pending_events: []
+     }}
+  end
+
+  @impl true
+  def handshake_data(%{clock: clock}), do: clock
+
+  @impl true
+  def handle_call({:put, key, pid, arg}, _from, state) do
+    %{store: store} = state
+
+    link(pid)
+
+    unless Store.present?(store, key, pid) do
+      case local_put(arg, key, pid, state) do
+        {:put, value, delta, state} ->
+          store = Store.local_put(store, key, pid, value, delta)
+          {:reply, :ok, %{state | store: store}}
+
+        {:delete, value, _delta} ->
+          # TODO: this delta has to propagate remotely before delete
+          state = local_delete([value], state)
+          {:reply, :ok, state}
+      end
+    else
+      {:reply, {:error, :already_present}, state}
+    end
+  end
+
+  def handle_call({:update, key, pid, arg}, _from, state) do
+    %{store: store} = state
+
+    case Store.fetch(store, key, pid) do
+      {:ok, value, delta} ->
+        case local_update(arg, key, pid, delta, value, state) do
+          {:put, value, delta, state} ->
+            store = Store.local_update(store, key, pid, value, delta)
+            {:reply, :ok, %{state | store: store}}
+
+          {:delete, value, _delta} ->
+            # TODO: this delta has to propagate remotely before delete
+            case Store.local_delete(store, key, pid) do
+              # The value returned from update is fresher
+              {:ok, _value, store} ->
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                state = local_delete([value], state)
+                {:reply, :ok, %{state | store: store}}
+
+              {:last_member, _value, store} ->
+                unlink_flush(pid)
+                state = local_delete([value], state)
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                {:reply, :ok, %{state | store: store}}
+            end
+        end
+
+      :error ->
+        {:reply, {:error, :not_present}, state}
+    end
+  end
+
+  def handle_call({:delete, key, pid}, _from, state) do
+    %{store: store} = state
+
+    case Store.local_delete(store, key, pid) do
+      {:ok, value, store} ->
+        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+        state = local_delete([value], state)
+        {:reply, :ok, %{state | store: store}}
+
+      {:last_member, value, store} ->
+        unlink_flush(pid)
+        state = local_delete([value], state)
+        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+        {:reply, :ok, %{state | store: store}}
+
+      {:error, store} ->
+        {:reply, {:error, :not_present}, %{state | store: store}}
+    end
+  end
+
+  def handle_call({:delete, pid}, _from, state) do
+    %{store: store} = state
+
+    case Store.local_delete(store, pid) do
+      {:ok, leaves, store} ->
+        unlink_flush(pid)
+        state = local_delete(leaves, state)
+        # state = schedule_broadcast_events(state, leaves)
+        {:reply, :ok, %{state | store: store}}
+
+      {:error, store} ->
+        {:reply, {:error, :not_member}, %{state | store: store}}
+    end
+  end
+
+  def handle_call({:list, key}, _from, state) do
+    %{store: store} = state
+
+    read = fn -> Store.list(store, key) end
+
+    {:reply, read, state}
+  end
+
+  @impl true
+  def handle_info({:EXIT, pid, reason}, state) do
+    %{store: store} = state
+
+    case Store.local_delete(store, pid) do
+      {:ok, leaves, store} ->
+        state = local_delete(leaves, state)
+        # state = schedule_broadcast_events(state, leaves)
+        {:noreply, %{state | store: store}}
+
+      {:error, store} ->
+        {:stop, reason, %{state | store: store}}
+    end
+  end
+
+  def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
+    %{pending_events: events, clock: clock} = state
+    clock = clock + 1
+    SyncedServer.remote_broadcast({:events, clock, events})
+    {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
+  end
+
+  def handle_info({:update, key, pid, arg}, state) do
+    %{store: store} = state
+
+    case Store.fetch(store, key, pid) do
+      {:ok, value, delta} ->
+        case local_update(arg, key, pid, delta, value, state) do
+          {:put, value, delta, state} ->
+            store = Store.local_update(store, key, pid, value, delta)
+            {:noreply, %{state | store: store}}
+
+          {:delete, value, _delta} ->
+            # TODO: this delta has to propagate remotely before delete
+            case Store.local_delete(store, key, pid) do
+              # The value returned from update is fresher
+              {:ok, _value, store} ->
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                state = local_delete([value], state)
+                {:noreply, %{state | store: store}}
+
+              {:last_member, _value, store} ->
+                unlink_flush(pid)
+                state = local_delete([value], state)
+                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                {:noreply, %{state | store: store}}
+            end
+        end
+
+      :error ->
+        # Must have been already deleted, ignore
+        {:noreply, state}
+    end
+  end
+
+  # @impl true
+  # def handle_remote({:catch_up_req, clock}, from, state) do
+  #   {mode, data} = catch_up_reply(state, clock)
+  #   SyncedServer.remote_send(from, {:catch_up, mode, data})
+  #   {:noreply, state}
+  # end
+
+  # def handle_remote({:catch_up, :state_transfer, {clock, transfer}}, from, state) do
+  #   state = handle_state_transfer(state, from, clock, transfer)
+  #   {:noreply, state}
+  # end
+
+  # def handle_remote({:events, remote_clock, events}, from, state) do
+  #   %{remote_clocks: remote_clocks} = state
+  #   local_clock = Map.fetch!(remote_clocks, from)
+
+  #   if remote_clock == local_clock + 1 do
+  #     remote_clocks = %{remote_clocks | from => remote_clock}
+  #     state = handle_events(state, from, events)
+  #     {:noreply, %{state | remote_clocks: remote_clocks}}
+  #   else
+  #     {:noreply, request_catch_up(state, from, local_clock)}
+  #   end
+  # end
+
+  # @impl true
+  # def handle_replica({:up, remote_clock}, remote_ref, state) do
+  #   %{remote_clocks: remote_clocks} = state
+
+  #   case remote_clocks do
+  #     %{^remote_ref => old_clock} when remote_clock > old_clock ->
+  #       # Reconnection, try to catch up
+  #       {:noreply, request_catch_up(state, remote_ref, old_clock)}
+
+  #     %{^remote_ref => old_clock} ->
+  #       # Reconnection, no remote state change, skip catch up
+  #       # Assert for sanity
+  #       true = old_clock == remote_clock
+  #       {:noreply, state}
+
+  #     %{} when remote_clock == 0 ->
+  #       # New node, no state, don't catch up
+  #       state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
+  #       {:noreply, state}
+
+  #     %{} ->
+  #       # New node, catch up
+  #       state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
+  #       {:noreply, request_catch_up(state, remote_ref, 0)}
+  #   end
+  # end
+
+  # def handle_replica(:down, remote_ref, state) do
+  #   %{values: values, remote_clocks: remote_clocks} = state
+  #   delete_ms = [{{:_, remote_ref, :_}, [], [true]}]
+  #   :ets.select_delete(values, delete_ms)
+  #   {:noreply, %{state | remote_clocks: Map.delete(remote_clocks, remote_ref)}}
+  # end
+
+  defp local_put(arg, key, pid, state) do
+    %{handler: handler, config: config, initial_delta: delta} = state
+
+    case handler.local_put(arg, delta, config) do
+      {delta, value} ->
+        {:put, value, delta, state}
+
+      {delta, value, :delete} ->
+        {:delete, value, delta}
+
+      {delta, value, {:update_after, update, time}} ->
+        Process.send_after(self(), {:update, key, pid, update}, time)
+        {:put, value, delta, state}
+    end
+  end
+
+  defp local_update(arg, key, pid, local_delta, value, state) do
+    %{handler: handler, config: config} = state
+
+    case handler.local_update(arg, local_delta, value, config) do
+      {delta, value} ->
+        {:put, value, delta, state}
+
+      {delta, value, :delete} ->
+        {:delete, value, delta}
+
+      {delta, value, {:update_after, update, time}} ->
+        Process.send_after(self(), {:update, key, pid, update}, time)
+        {:put, value, delta, state}
+    end
+  end
+
+  defp local_delete(leaves, state) do
+    %{handler: handler, config: config} = state
+
+    Enum.each(leaves, &handler.local_delete(&1, config))
+    state
+  end
+
+  defp link(:partition), do: true
+  defp link(pid), do: Process.link(pid)
+
+  defp unlink_flush(:partition), do: true
+
+  defp unlink_flush(pid) do
+    Process.unlink(pid)
+
+    receive do
+      {:EXIT, ^pid, _} -> true
+    after
+      0 -> true
+    end
+  end
+
+  defp schedule_broadcast_events(%{broadcast_timer: nil} = state, new_events) do
+    %{broadcast_timeout: timeout, pending_events: events} = state
+    timer = :erlang.start_timer(timeout, self(), :broadcast)
+    %{state | broadcast_timer: timer, pending_events: new_events ++ events}
+  end
+
+  defp schedule_broadcast_events(%{} = state, new_events) do
+    %{pending_events: events} = state
+    %{state | pending_events: new_events ++ events}
+  end
+
+  defp request_catch_up(state, remote_ref, clock) do
+    SyncedServer.remote_send(remote_ref, {:catch_up_req, clock})
+    state
+  end
+
+  defp handle_events(%{values: values} = state, from, events) do
+    {joins, leaves} =
+      Enum.reduce(events, {[], []}, fn
+        {:leave, key, pid}, {joins, leaves} ->
+          leave = {{{key, pid}, from, :_}, [], [true]}
+          {joins, [leave | leaves]}
+
+        {:replace, key, pid, value}, {joins, leaves} ->
+          join = {{key, pid}, from, value}
+          {[join | joins], leaves}
+
+        {:join, key, pid, value}, {joins, leaves} ->
+          join = {{key, pid}, from, value}
+          {[join | joins], leaves}
+      end)
+
+    :ets.insert(values, joins)
+    :ets.select_delete(values, leaves)
+    state
+  end
+
+  # TODO: detect leaves
+  # Is there a better way than to clean up and re-insert?
+  # This can be problematic for dirty reads!
+  defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
+    %{remote_clocks: remote_clocks} = state
+    delete_ms = [{{:_, from, :_}, [], [true]}]
+    inserts = for {ets_key, value} <- transfer, do: {ets_key, from, value}
+    :ets.select_delete(values, delete_ms)
+    :ets.insert(values, inserts)
+    %{state | remote_clocks: %{remote_clocks | from => clock}}
+  end
+
+  # TODO: handle catch-up with events
+  defp catch_up_reply(%{values: values}, clock) do
+    local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
+    {:state_transfer, {clock, :ets.select(values, local_ms)}}
+  end
+end

From 935373af84cd2b036ddf94737fed16b73a87e0ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 30 Oct 2018 13:21:40 +0100
Subject: [PATCH 22/40] wip

---
 lib/firenest/replicated_state/remote.ex | 89 +++++++++++++++++++++++++
 lib/firenest/replicated_state/server.ex |  7 +-
 lib/firenest/replicated_state/store.ex  |  4 +-
 3 files changed, 95 insertions(+), 5 deletions(-)
 create mode 100644 lib/firenest/replicated_state/remote.ex

diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
new file mode 100644
index 0000000..053a3d3
--- /dev/null
+++ b/lib/firenest/replicated_state/remote.ex
@@ -0,0 +1,89 @@
+defmodule Firenest.ReplicatedState.Remote do
+  defstruct pending: %{}, clocks: %{}, clock: 0, event: nil, tag: nil
+
+  def new(:ignore) do
+    %__MODULE__{event: &event_ignore/3, tag: :ignore}
+  end
+
+  # Reconnections are dead until we have permdown
+  def up(%__MODULE__{clocks: clocks} = state, ref, clock) do
+    case clocks do
+      # Reconnection, try to catch up
+      %{^ref => old_clock} when clock > old_clock ->
+        {:catch_up, ref, clock, old_clock, state}
+
+      # Reconnection, no remote changes
+      %{^ref => old_clock} ->
+        # Assert for sanity
+        true = old_clock == clock
+        {:ok, state}
+
+      # New node, no state
+      %{} when clock == 0 ->
+        {:ok, %{state | clocks: Map.put(clocks, ref, clock)}}
+
+      # New node, catch up
+      %{} ->
+        {:catch_up, ref, clock, 0, state}
+    end
+  end
+
+  def down()
+
+  def catch_up()
+
+  def broadcast(%__MODULE__{pending: pending, clock: clock, tag: tag} = state) do
+    new_state = %{state | pending: %{}, clock: clock + 1}
+    {{tag, clock, Map.to_list(pending)}, new_state}
+  end
+
+  def handle_broadcast(%__MODULE__{clocks: clocks, tag: tag}, ref, {tag, clock, data}) do
+    handle_broadcast(clocks, ref, clock, data)
+  end
+
+  def handle_broadcast(%__MODULE__{tag: local_tag}, ref, {remote_tag, _, _}) do
+    {:error, {:different_tag, ref, local_tag, remote_tag}}
+  end
+
+  def handle_catch_up()
+
+  def local_put(state, key) do
+    event(state, key, :put)
+  end
+
+  def local_delete(state, key) do
+    event(state, key, :delete)
+  end
+
+  def local_update(state, key) do
+    event(state, key, :update)
+  end
+
+  defp handle_broadcast(clocks, ref, clock, data) do
+
+  end
+
+  defp event(%__MODULE__{pending: pending, event: process} = state, key, event) do
+    pending = process.(pending, key, event)
+    %{state | pending: pending}
+  end
+
+  def event_ignore(pending, key, :put) do
+    Map.put(pending, key, :put)
+  end
+
+  def event_ignore(pending, key, :delete) do
+    case pending do
+      %{^key => :put} -> Map.delete(pending, key)
+      %{} -> Map.put(pending, key, :delete)
+    end
+  end
+
+  def event_ignore(pending, key, :update) do
+    case pending do
+      %{^key => :put} -> pending
+      %{^key => :update} -> pending
+      %{} -> Map.put(pending, key, :update)
+    end
+  end
+end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 1be2e2b..6533441 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -3,7 +3,7 @@ defmodule Firenest.ReplicatedState.Server do
   use Firenest.SyncedServer
 
   alias Firenest.SyncedServer
-  alias Firenest.ReplicatedState.Store
+  alias Firenest.ReplicatedState.{Store, Remote}
 
   def child_spec({name, topology, handler, opts}) do
     server_opts = [name: name, topology: topology]
@@ -19,7 +19,8 @@ defmodule Firenest.ReplicatedState.Server do
     Process.flag(:trap_exit, true)
     store = Store.new(name)
     broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
-
+    remote_changes = Keyword.get(opts, :remote_changes, :ignore)
+    remote = Remote.new(remote_changes)
     {initial_delta, config} = handler.init(opts)
 
     {:ok,
@@ -30,8 +31,8 @@ defmodule Firenest.ReplicatedState.Server do
        handler: handler,
        broadcast_timer: nil,
        broadcast_timeout: broadcast_timeout,
+       remote: remote,
        clock: 0,
-       remote_clocks: %{},
        pending_events: []
      }}
   end
diff --git a/lib/firenest/replicated_state/store.ex b/lib/firenest/replicated_state/store.ex
index aa7616d..30f5720 100644
--- a/lib/firenest/replicated_state/store.ex
+++ b/lib/firenest/replicated_state/store.ex
@@ -2,8 +2,8 @@ defmodule Firenest.ReplicatedState.Store do
   defstruct [:values, :pids]
 
   def new(name) do
-    values = :ets.new(name, [:named_table, :protected, :ordered_set])
-    pids = :ets.new(__MODULE__.Pids, [:duplicate_bag, keypos: 2])
+    values = :ets.new(name, [:named_table, :protected, :ordered_set, read_concurrency: true])
+    pids = :ets.new(__MODULE__.Pids, [:private, :duplicate_bag, keypos: 2])
 
     %__MODULE__{values: ets_whereis(values), pids: pids}
   end

From 7461cd9c8d191da0c2e22c13d6b482d274103a42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 5 Nov 2018 13:56:54 +0100
Subject: [PATCH 23/40] Extract handler code from ReplicatedState.Server

---
 lib/firenest/replicated_state/handler.ex |  47 ++++++++
 lib/firenest/replicated_state/server.ex  | 141 ++++++++---------------
 2 files changed, 98 insertions(+), 90 deletions(-)
 create mode 100644 lib/firenest/replicated_state/handler.ex

diff --git a/lib/firenest/replicated_state/handler.ex b/lib/firenest/replicated_state/handler.ex
new file mode 100644
index 0000000..8c8a048
--- /dev/null
+++ b/lib/firenest/replicated_state/handler.ex
@@ -0,0 +1,47 @@
+defmodule Firenest.ReplicatedState.Handler do
+  defstruct mod: nil, init_delta: nil, config: nil, delayed_fun: nil
+
+  def new(mod, mod_opts, delayed_fun) do
+    {init_delta, config} = mod.init(mod_opts)
+    %__MODULE__{mod: mod, init_delta: init_delta, config: config, delayed_fun: delayed_fun}
+  end
+
+  def local_put(%__MODULE__{} = state, arg, key, pid) do
+    %{mod: mod, config: config, init_delta: delta, delayed_fun: delayed} = state
+
+    case mod.local_put(arg, delta, config) do
+      {delta, value} ->
+        {:put, value, delta, state}
+
+      {delta, value, :delete} ->
+        {:delete, value, delta, state}
+
+      {delta, value, {:update_after, update, time}} ->
+        delayed.(key, pid, update, time)
+        {:put, value, delta, state}
+    end
+  end
+
+  def local_update(%__MODULE__{} = state, arg, key, pid, local_delta, value) do
+    %{mod: mod, config: config, delayed_fun: delayed} = state
+
+    case mod.local_update(arg, local_delta, value, config) do
+      {delta, value} ->
+        {:put, value, delta, state}
+
+      {delta, value, :delete} ->
+        {:delete, value, delta, state}
+
+      {delta, value, {:update_after, update, time}} ->
+        delayed.(key, pid, update, time)
+        {:put, value, delta, state}
+    end
+  end
+
+  def local_delete(%__MODULE__{} = state, deletes) do
+    %{mod: mod, config: config} = state
+
+    Enum.each(deletes, &mod.local_delete(&1, config))
+    state
+  end
+end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 6533441..7d119f2 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -3,7 +3,7 @@ defmodule Firenest.ReplicatedState.Server do
   use Firenest.SyncedServer
 
   alias Firenest.SyncedServer
-  alias Firenest.ReplicatedState.{Store, Remote}
+  alias Firenest.ReplicatedState.{Store, Remote, Handler}
 
   def child_spec({name, topology, handler, opts}) do
     server_opts = [name: name, topology: topology]
@@ -17,45 +17,45 @@ defmodule Firenest.ReplicatedState.Server do
   @impl true
   def init({name, handler, opts}) do
     Process.flag(:trap_exit, true)
+
     store = Store.new(name)
+
     broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
     remote_changes = Keyword.get(opts, :remote_changes, :ignore)
     remote = Remote.new(remote_changes)
-    {initial_delta, config} = handler.init(opts)
+
+    delayed_fun = &Process.send_after(self(), {:update, &1, &2, &3}, &4)
+    handler = Handler.new(handler, opts, delayed_fun)
 
     {:ok,
      %{
        store: store,
-       config: config,
-       initial_delta: initial_delta,
        handler: handler,
+       remote: remote,
        broadcast_timer: nil,
        broadcast_timeout: broadcast_timeout,
-       remote: remote,
-       clock: 0,
-       pending_events: []
      }}
   end
 
   @impl true
-  def handshake_data(%{clock: clock}), do: clock
+  def handshake_data(%{remote: remote}), do: Remote.clock(remote)
 
   @impl true
   def handle_call({:put, key, pid, arg}, _from, state) do
-    %{store: store} = state
+    %{store: store, handler: handler} = state
 
     link(pid)
 
     unless Store.present?(store, key, pid) do
-      case local_put(arg, key, pid, state) do
-        {:put, value, delta, state} ->
+      case Handler.local_put(handler, arg, key, pid) do
+        {:put, value, delta, handler} ->
           store = Store.local_put(store, key, pid, value, delta)
-          {:reply, :ok, %{state | store: store}}
+          {:reply, :ok, %{state | store: store, handler: handler}}
 
-        {:delete, value, _delta} ->
+        {:delete, value, _delta, handler} ->
           # TODO: this delta has to propagate remotely before delete
-          state = local_delete([value], state)
-          {:reply, :ok, state}
+          handler = Handler.local_delete(handler, [value])
+          {:reply, :ok, %{state | handler: handler}}
       end
     else
       {:reply, {:error, :already_present}, state}
@@ -63,29 +63,29 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:update, key, pid, arg}, _from, state) do
-    %{store: store} = state
+    %{store: store, handler: handler} = state
 
     case Store.fetch(store, key, pid) do
       {:ok, value, delta} ->
-        case local_update(arg, key, pid, delta, value, state) do
-          {:put, value, delta, state} ->
+        case Handler.local_update(handler, arg, key, pid, delta, value) do
+          {:put, value, delta, handler} ->
             store = Store.local_update(store, key, pid, value, delta)
-            {:reply, :ok, %{state | store: store}}
+            {:reply, :ok, %{state | store: store, handler: handler}}
 
-          {:delete, value, _delta} ->
+          {:delete, value, _delta, handler} ->
             # TODO: this delta has to propagate remotely before delete
             case Store.local_delete(store, key, pid) do
               # The value returned from update is fresher
               {:ok, _value, store} ->
                 # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                state = local_delete([value], state)
-                {:reply, :ok, %{state | store: store}}
+                handler = Handler.local_delete(handler, [value])
+                {:reply, :ok, %{state | store: store, handler: handler}}
 
               {:last_member, _value, store} ->
                 unlink_flush(pid)
-                state = local_delete([value], state)
+                handler = Handler.local_delete(handler, [value])
                 # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                {:reply, :ok, %{state | store: store}}
+                {:reply, :ok, %{state | store: store, handler: handler}}
             end
         end
 
@@ -95,19 +95,19 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:delete, key, pid}, _from, state) do
-    %{store: store} = state
+    %{store: store, handler: handler} = state
 
     case Store.local_delete(store, key, pid) do
       {:ok, value, store} ->
         # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-        state = local_delete([value], state)
-        {:reply, :ok, %{state | store: store}}
+        handler = Handler.local_delete(handler, [value])
+        {:reply, :ok, %{state | store: store, handler: handler}}
 
       {:last_member, value, store} ->
         unlink_flush(pid)
-        state = local_delete([value], state)
+        handler = Handler.local_delete(handler, [value])
         # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-        {:reply, :ok, %{state | store: store}}
+        {:reply, :ok, %{state | store: store, handler: handler}}
 
       {:error, store} ->
         {:reply, {:error, :not_present}, %{state | store: store}}
@@ -115,14 +115,14 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:delete, pid}, _from, state) do
-    %{store: store} = state
+    %{store: store, handler: handler} = state
 
     case Store.local_delete(store, pid) do
-      {:ok, leaves, store} ->
+      {:ok, deletes, store} ->
         unlink_flush(pid)
-        state = local_delete(leaves, state)
+        handler = Handler.local_delete(handler, deletes)
         # state = schedule_broadcast_events(state, leaves)
-        {:reply, :ok, %{state | store: store}}
+        {:reply, :ok, %{state | store: store, handler: handler}}
 
       {:error, store} ->
         {:reply, {:error, :not_member}, %{state | store: store}}
@@ -139,50 +139,50 @@ defmodule Firenest.ReplicatedState.Server do
 
   @impl true
   def handle_info({:EXIT, pid, reason}, state) do
-    %{store: store} = state
+    %{store: store, handler: handler} = state
 
     case Store.local_delete(store, pid) do
       {:ok, leaves, store} ->
-        state = local_delete(leaves, state)
+        handler = Handler.local_delete(handler, leaves)
         # state = schedule_broadcast_events(state, leaves)
-        {:noreply, %{state | store: store}}
+        {:noreply, %{state | store: store, handler: handler}}
 
       {:error, store} ->
         {:stop, reason, %{state | store: store}}
     end
   end
 
-  def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
-    %{pending_events: events, clock: clock} = state
-    clock = clock + 1
-    SyncedServer.remote_broadcast({:events, clock, events})
-    {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
-  end
+  # def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
+  #   %{pending_events: events, clock: clock} = state
+  #   clock = clock + 1
+  #   SyncedServer.remote_broadcast({:events, clock, events})
+  #   {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
+  # end
 
   def handle_info({:update, key, pid, arg}, state) do
-    %{store: store} = state
+    %{store: store, handler: handler} = state
 
     case Store.fetch(store, key, pid) do
       {:ok, value, delta} ->
-        case local_update(arg, key, pid, delta, value, state) do
-          {:put, value, delta, state} ->
+        case Handler.local_update(handler, arg, key, pid, delta, value) do
+          {:put, value, delta, handler} ->
             store = Store.local_update(store, key, pid, value, delta)
-            {:noreply, %{state | store: store}}
+            {:noreply, %{state | store: store, handler: handler}}
 
-          {:delete, value, _delta} ->
+          {:delete, value, _delta, handler} ->
             # TODO: this delta has to propagate remotely before delete
             case Store.local_delete(store, key, pid) do
               # The value returned from update is fresher
               {:ok, _value, store} ->
                 # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                state = local_delete([value], state)
-                {:noreply, %{state | store: store}}
+                handler = Handler.local_delete(handler, [value])
+                {:noreply, %{state | store: store, handler: handler}}
 
               {:last_member, _value, store} ->
                 unlink_flush(pid)
-                state = local_delete([value], state)
+                handler = Handler.local_delete(handler, [value])
                 # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                {:noreply, %{state | store: store}}
+                {:noreply, %{state | store: store, handler: handler}}
             end
         end
 
@@ -251,45 +251,6 @@ defmodule Firenest.ReplicatedState.Server do
   #   {:noreply, %{state | remote_clocks: Map.delete(remote_clocks, remote_ref)}}
   # end
 
-  defp local_put(arg, key, pid, state) do
-    %{handler: handler, config: config, initial_delta: delta} = state
-
-    case handler.local_put(arg, delta, config) do
-      {delta, value} ->
-        {:put, value, delta, state}
-
-      {delta, value, :delete} ->
-        {:delete, value, delta}
-
-      {delta, value, {:update_after, update, time}} ->
-        Process.send_after(self(), {:update, key, pid, update}, time)
-        {:put, value, delta, state}
-    end
-  end
-
-  defp local_update(arg, key, pid, local_delta, value, state) do
-    %{handler: handler, config: config} = state
-
-    case handler.local_update(arg, local_delta, value, config) do
-      {delta, value} ->
-        {:put, value, delta, state}
-
-      {delta, value, :delete} ->
-        {:delete, value, delta}
-
-      {delta, value, {:update_after, update, time}} ->
-        Process.send_after(self(), {:update, key, pid, update}, time)
-        {:put, value, delta, state}
-    end
-  end
-
-  defp local_delete(leaves, state) do
-    %{handler: handler, config: config} = state
-
-    Enum.each(leaves, &handler.local_delete(&1, config))
-    state
-  end
-
   defp link(:partition), do: true
   defp link(pid), do: Process.link(pid)
 

From bfc0218a1a9497d012f15f0cf74411702cf5f9e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 5 Nov 2018 19:12:37 +0100
Subject: [PATCH 24/40] wip remote replicated state

---
 lib/firenest/replicated_state/remote.ex | 105 +++++++++++++++++-------
 lib/firenest/replicated_state/server.ex |  99 +++++++++++-----------
 2 files changed, 124 insertions(+), 80 deletions(-)

diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index 053a3d3..36139e0 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -1,16 +1,18 @@
 defmodule Firenest.ReplicatedState.Remote do
-  defstruct pending: %{}, clocks: %{}, clock: 0, event: nil, tag: nil
+  defstruct pending: %{}, clocks: %{}, clock: 0, tag: nil, deltas: %{}
 
   def new(:ignore) do
-    %__MODULE__{event: &event_ignore/3, tag: :ignore}
+    %__MODULE__{tag: :ignore}
   end
 
+  def clock(%__MODULE__{clock: clock}), do: clock
+
   # Reconnections are dead until we have permdown
   def up(%__MODULE__{clocks: clocks} = state, ref, clock) do
     case clocks do
       # Reconnection, try to catch up
       %{^ref => old_clock} when clock > old_clock ->
-        {:catch_up, ref, clock, old_clock, state}
+        {:catch_up, {clock, old_clock}, state}
 
       # Reconnection, no remote changes
       %{^ref => old_clock} ->
@@ -24,66 +26,107 @@ defmodule Firenest.ReplicatedState.Remote do
 
       # New node, catch up
       %{} ->
-        {:catch_up, ref, clock, 0, state}
+        {:catch_up, {clock, 0}, state}
     end
   end
 
-  def down()
+  # Right now down means permdown
+  def down(state, ref) do
+    permdown(state, ref)
+  end
+
+  def permdown(%__MODULE__{clocks: clocks} = state, ref) do
+    clocks = Map.delete(clocks, ref)
+    {:purge, ref, %{state | clocks: clocks}}
+  end
+
+  def catch_up(%__MODULE__{clock: current} = state, {clock, old_clock}, state_getter)
+      when old_clock < clock and clock <= current do
+    %{deltas: deltas, tag: tag} = state
 
-  def catch_up()
+    if Map.has_key?(deltas, old_clock) do
+      {:deltas, tag, Enum.flat_map(old_clock..clock, &Map.fetch!(deltas, &1))}
+    else
+      {:transfer_state, {:state_transfer, tag, current, state_getter.()}}
+    end
+  end
 
-  def broadcast(%__MODULE__{pending: pending, clock: clock, tag: tag} = state) do
+  def broadcast(%__MODULE__{pending: pending, clock: clock, tag: tag} = state, state_getter) do
+    deltas = prepare_deltas(tag, pending, state_getter)
     new_state = %{state | pending: %{}, clock: clock + 1}
-    {{tag, clock, Map.to_list(pending)}, new_state}
+    {{tag, clock, deltas}, new_state}
+  end
+
+  def handle_catch_up(%__MODULE__{tag: tag} = state, from, {:deltas, tag, deltas}) do
+  end
+
+  def handle_catch_up(%__MODULE__{tag: tag} = state, from, {:state_transfer, tag, clock, state}) do
   end
 
   def handle_broadcast(%__MODULE__{clocks: clocks, tag: tag}, ref, {tag, clock, data}) do
-    handle_broadcast(clocks, ref, clock, data)
+    # handle_broadcast(tag, clocks, ref, clock, data)
   end
 
   def handle_broadcast(%__MODULE__{tag: local_tag}, ref, {remote_tag, _, _}) do
     {:error, {:different_tag, ref, local_tag, remote_tag}}
   end
 
-  def handle_catch_up()
-
-  def local_put(state, key) do
-    event(state, key, :put)
+  def local_put(state, key, pid) do
+    event(state, key, pid, :put)
   end
 
-  def local_delete(state, key) do
-    event(state, key, :delete)
+  def local_delete(state, key, pid) do
+    event(state, key, pid, :delete)
   end
 
-  def local_update(state, key) do
-    event(state, key, :update)
+  def local_update(state, key, pid) do
+    event(state, key, pid, :update)
   end
 
-  defp handle_broadcast(clocks, ref, clock, data) do
+  defp event(%__MODULE__{pending: pending, tag: tag} = state, key, pid, event) do
+    pending =
+      case tag do
+        :ignore -> event_ignore(pending, key, pid, event)
+      end
 
-  end
-
-  defp event(%__MODULE__{pending: pending, event: process} = state, key, event) do
-    pending = process.(pending, key, event)
     %{state | pending: pending}
   end
 
-  def event_ignore(pending, key, :put) do
-    Map.put(pending, key, :put)
+  def event_ignore(pending, key, pid, :put) do
+    Map.put(pending, {key, pid}, :put)
   end
 
-  def event_ignore(pending, key, :delete) do
+  def event_ignore(pending, key, pid, :delete) do
+    pending_key = {key, pid}
+
     case pending do
-      %{^key => :put} -> Map.delete(pending, key)
-      %{} -> Map.put(pending, key, :delete)
+      %{^pending_key => :put} -> Map.delete(pending, pending_key)
+      %{} -> Map.put(pending, pending_key, :delete)
     end
   end
 
-  def event_ignore(pending, key, :update) do
+  def event_ignore(pending, key, pid, :update) do
+    pending_key = {key, pid}
+
     case pending do
-      %{^key => :put} -> pending
-      %{^key => :update} -> pending
-      %{} -> Map.put(pending, key, :update)
+      %{^pending_key => :put} -> pending
+      %{} -> Map.put(pending, pending_key, :update)
     end
   end
+
+  defp prepare_deltas(:ignore, pending, getter) do
+    Enum.map(pending, &prepare_ignore_delta(&1, getter))
+  end
+
+  defp prepare_ignore_delta({{key, pid}, :put}, getter) do
+    {:put, key, getter.(key, pid)}
+  end
+
+  defp prepare_ignore_delta({{key, pid}, :update}, getter) do
+    {:put, key, getter.(key, pid)}
+  end
+
+  defp prepare_ignore_delta({{key, _pid}, :delete}, _getter) do
+    {:delete, key}
+  end
 end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 7d119f2..b8977b1 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -25,6 +25,7 @@ defmodule Firenest.ReplicatedState.Server do
     remote = Remote.new(remote_changes)
 
     delayed_fun = &Process.send_after(self(), {:update, &1, &2, &3}, &4)
+    # The `mode` should be read from init instead of options
     handler = Handler.new(handler, opts, delayed_fun)
 
     {:ok,
@@ -266,58 +267,58 @@ defmodule Firenest.ReplicatedState.Server do
     end
   end
 
-  defp schedule_broadcast_events(%{broadcast_timer: nil} = state, new_events) do
-    %{broadcast_timeout: timeout, pending_events: events} = state
-    timer = :erlang.start_timer(timeout, self(), :broadcast)
-    %{state | broadcast_timer: timer, pending_events: new_events ++ events}
-  end
+  # defp schedule_broadcast_events(%{broadcast_timer: nil} = state, new_events) do
+  #   %{broadcast_timeout: timeout, pending_events: events} = state
+  #   timer = :erlang.start_timer(timeout, self(), :broadcast)
+  #   %{state | broadcast_timer: timer, pending_events: new_events ++ events}
+  # end
 
-  defp schedule_broadcast_events(%{} = state, new_events) do
-    %{pending_events: events} = state
-    %{state | pending_events: new_events ++ events}
-  end
+  # defp schedule_broadcast_events(%{} = state, new_events) do
+  #   %{pending_events: events} = state
+  #   %{state | pending_events: new_events ++ events}
+  # end
 
-  defp request_catch_up(state, remote_ref, clock) do
-    SyncedServer.remote_send(remote_ref, {:catch_up_req, clock})
-    state
-  end
+  # defp request_catch_up(state, remote_ref, clock) do
+  #   SyncedServer.remote_send(remote_ref, {:catch_up_req, clock})
+  #   state
+  # end
 
-  defp handle_events(%{values: values} = state, from, events) do
-    {joins, leaves} =
-      Enum.reduce(events, {[], []}, fn
-        {:leave, key, pid}, {joins, leaves} ->
-          leave = {{{key, pid}, from, :_}, [], [true]}
-          {joins, [leave | leaves]}
-
-        {:replace, key, pid, value}, {joins, leaves} ->
-          join = {{key, pid}, from, value}
-          {[join | joins], leaves}
-
-        {:join, key, pid, value}, {joins, leaves} ->
-          join = {{key, pid}, from, value}
-          {[join | joins], leaves}
-      end)
-
-    :ets.insert(values, joins)
-    :ets.select_delete(values, leaves)
-    state
-  end
+  # defp handle_events(%{values: values} = state, from, events) do
+  #   {joins, leaves} =
+  #     Enum.reduce(events, {[], []}, fn
+  #       {:leave, key, pid}, {joins, leaves} ->
+  #         leave = {{{key, pid}, from, :_}, [], [true]}
+  #         {joins, [leave | leaves]}
+
+  #       {:replace, key, pid, value}, {joins, leaves} ->
+  #         join = {{key, pid}, from, value}
+  #         {[join | joins], leaves}
+
+  #       {:join, key, pid, value}, {joins, leaves} ->
+  #         join = {{key, pid}, from, value}
+  #         {[join | joins], leaves}
+  #     end)
+
+  #   :ets.insert(values, joins)
+  #   :ets.select_delete(values, leaves)
+  #   state
+  # end
 
-  # TODO: detect leaves
-  # Is there a better way than to clean up and re-insert?
-  # This can be problematic for dirty reads!
-  defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
-    %{remote_clocks: remote_clocks} = state
-    delete_ms = [{{:_, from, :_}, [], [true]}]
-    inserts = for {ets_key, value} <- transfer, do: {ets_key, from, value}
-    :ets.select_delete(values, delete_ms)
-    :ets.insert(values, inserts)
-    %{state | remote_clocks: %{remote_clocks | from => clock}}
-  end
+  # # TODO: detect leaves
+  # # Is there a better way than to clean up and re-insert?
+  # # This can be problematic for dirty reads!
+  # defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
+  #   %{remote_clocks: remote_clocks} = state
+  #   delete_ms = [{{:_, from, :_}, [], [true]}]
+  #   inserts = for {ets_key, value} <- transfer, do: {ets_key, from, value}
+  #   :ets.select_delete(values, delete_ms)
+  #   :ets.insert(values, inserts)
+  #   %{state | remote_clocks: %{remote_clocks | from => clock}}
+  # end
 
-  # TODO: handle catch-up with events
-  defp catch_up_reply(%{values: values}, clock) do
-    local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
-    {:state_transfer, {clock, :ets.select(values, local_ms)}}
-  end
+  # # TODO: handle catch-up with events
+  # defp catch_up_reply(%{values: values}, clock) do
+  #   local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
+  #   {:state_transfer, {clock, :ets.select(values, local_ms)}}
+  # end
 end

From 0182bafcfab19ac5043cca9c83dde7a02ad80109 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 12 Nov 2018 14:15:34 +0100
Subject: [PATCH 25/40] First remote implementation for replicatedstate

---
 lib/firenest/replicated_state.ex         |  36 ++++---
 lib/firenest/replicated_state/handler.ex |   6 ++
 lib/firenest/replicated_state/remote.ex  | 111 ++++++++++++++------
 lib/firenest/replicated_state/server.ex  | 123 +++++++++++++----------
 lib/firenest/replicated_state/store.ex   |  42 +++++++-
 5 files changed, 218 insertions(+), 100 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index a95ee71..4a77cc2 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -30,20 +30,28 @@ defmodule Firenest.ReplicatedState do
   @type local_delta() :: term()
   @type remote_delta() :: term()
   @type state() :: term()
-  @type config() :: term()
+  @type callback_config() :: term()
 
   @type extra_action :: :delete | {:update_after, update :: term(), time :: pos_integer()}
 
+  @type server_opt ::
+    {:remote_changes, :ignore | :observe_collapsed | :observe_full}
+
   @doc """
   Called when a partition starts up.
 
-  It returns an `initial_delta` that will be passed to `c:local_put/3`
-  callback on new entries and to `c:local_update/4` after remote
-  broadcast resets the local delta.
-  It also returns an immutable `config` value that will be passed to
-  all callbacks.
+  It returns:
+
+    * an `initial_delta` that will be passed to `c:local_put/3`
+      callback on new entries and to `c:local_update/4` after remote
+      broadcast resets the local delta.
+    * an immutable `callback_config` value that will be passed to all
+      callbacks.
+    * a list of server_opt` settings configuring the behaviour of the server
+
   """
-  @callback init(opts :: keyword()) :: {initial_delta :: local_delta(), config()}
+  @callback init(opts :: keyword()) ::
+              {initial_delta :: local_delta(), callback_config(), [server_opt]}
 
   @doc """
   Called whenever the `put/4` function is called to create a new state.
@@ -57,7 +65,7 @@ defmodule Firenest.ReplicatedState do
 
   This is a good place for broadcasting local state changes.
   """
-  @callback local_put(arg :: term(), local_delta(), config()) ::
+  @callback local_put(arg :: term(), local_delta(), callback_config()) ::
               {local_delta(), initial_state} | {local_delta(), initial_state, extra_action()}
             when initial_state: state()
 
@@ -80,7 +88,7 @@ defmodule Firenest.ReplicatedState do
   If the third element is `:delete`, the state will be immediately
   deleted and the `c:local_delete/2` callback triggered.
   """
-  @callback local_update(update :: term(), local_delta(), state(), config()) ::
+  @callback local_update(update :: term(), local_delta(), state(), callback_config()) ::
               {local_delta(), state()} | {local_delta(), state(), extra_action()}
 
   @doc """
@@ -91,7 +99,7 @@ defmodule Firenest.ReplicatedState do
 
   This is a good place for broadcasting local state changes.
   """
-  @callback local_delete(state(), config()) :: term()
+  @callback local_delete(state(), callback_config()) :: term()
 
   @doc """
   Called whenever the server is about to incrementally replicate local
@@ -104,7 +112,7 @@ defmodule Firenest.ReplicatedState do
 
   In case the callback is not provided it defaults to just returning local delta.
   """
-  @callback prepare_remote_delta(local_delta(), config()) :: remote_delta()
+  @callback prepare_remote_delta(local_delta(), callback_config()) :: remote_delta()
 
   @doc """
   Called whenever a remote delta is received from another node.
@@ -114,7 +122,7 @@ defmodule Firenest.ReplicatedState do
   exactly the same as the result of applying local updates to the
   state in the `c:local_update/3` callback.
   """
-  @callback handle_remote_delta(remote_delta(), state(), config()) :: state()
+  @callback handle_remote_delta(remote_delta(), state(), callback_config()) :: state()
 
   @doc """
   Called when remote changes are received by the local server.
@@ -132,7 +140,7 @@ defmodule Firenest.ReplicatedState do
       incremental changes to the remote state were communicated.
 
   This callback is optional and its behaviour depends on the value
-  of the `:remote_changes` option provided when the server is started.
+  of the `:remote_changes` option returned from the `c:init/2` callback.
 
     * `:ignore` - the callback is not invoked and the server skips
       all operations required for tracking the changes. This is the
@@ -155,7 +163,7 @@ defmodule Firenest.ReplicatedState do
 
   The return value is ignored.
   """
-  @callback observe_remote_changes(observed_remote_changes, config()) :: term()
+  @callback observe_remote_changes(observed_remote_changes, callback_config()) :: term()
             when observed_remote_changes: [{key(), [process_state_change]}],
                  process_state_change: {current_state :: state(), [change]},
                  change:
diff --git a/lib/firenest/replicated_state/handler.ex b/lib/firenest/replicated_state/handler.ex
index 8c8a048..16d64e2 100644
--- a/lib/firenest/replicated_state/handler.ex
+++ b/lib/firenest/replicated_state/handler.ex
@@ -44,4 +44,10 @@ defmodule Firenest.ReplicatedState.Handler do
     Enum.each(deletes, &mod.local_delete(&1, config))
     state
   end
+
+  def handle_remote_delta(%__MODULE__{} = state, delta, value) do
+    %{mod: mod, config: config} = state
+
+    mod.handle_remote_delta(delta, value, config)
+  end
 end
diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index 36139e0..0f2508e 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -1,6 +1,9 @@
 defmodule Firenest.ReplicatedState.Remote do
   defstruct pending: %{}, clocks: %{}, clock: 0, tag: nil, deltas: %{}
 
+  # TODO: some protocol for requesting more of a state, even from other nodes
+  # on first up, so we don't need to go to each node separately.
+
   def new(:ignore) do
     %__MODULE__{tag: :ignore}
   end
@@ -30,14 +33,15 @@ defmodule Firenest.ReplicatedState.Remote do
     end
   end
 
-  # Right now down means permdown
+  # TODO: Right now down means permdown
   def down(state, ref) do
     permdown(state, ref)
   end
 
   def permdown(%__MODULE__{clocks: clocks} = state, ref) do
+    true = Map.has_key?(clocks, ref)
     clocks = Map.delete(clocks, ref)
-    {:purge, ref, %{state | clocks: clocks}}
+    {:delete, ref, %{state | clocks: clocks}}
   end
 
   def catch_up(%__MODULE__{clock: current} = state, {clock, old_clock}, state_getter)
@@ -45,42 +49,82 @@ defmodule Firenest.ReplicatedState.Remote do
     %{deltas: deltas, tag: tag} = state
 
     if Map.has_key?(deltas, old_clock) do
-      {:deltas, tag, Enum.flat_map(old_clock..clock, &Map.fetch!(deltas, &1))}
+      {:deltas, tag, current, Enum.flat_map(old_clock..current, &Map.fetch!(deltas, &1))}
     else
-      {:transfer_state, {:state_transfer, tag, current, state_getter.()}}
+      {:state_transfer, tag, current, state_getter.()}
     end
   end
 
-  def broadcast(%__MODULE__{pending: pending, clock: clock, tag: tag} = state, state_getter) do
-    deltas = prepare_deltas(tag, pending, state_getter)
+  def broadcast(%__MODULE__{pending: pending, clock: clock, tag: tag} = state, prepare_delta) do
+    deltas = prepare_deltas(tag, pending, prepare_delta)
     new_state = %{state | pending: %{}, clock: clock + 1}
     {{tag, clock, deltas}, new_state}
   end
 
-  def handle_catch_up(%__MODULE__{tag: tag} = state, from, {:deltas, tag, deltas}) do
+  def handle_catch_up(%__MODULE__{tag: tag} = state, ref, {:deltas, tag, clock, deltas}) do
+    %{clocks: clocks} = state
+
+    state = %{state | clocks: %{clocks | ref => clock}}
+    {puts, updates, deletes} = handle_deltas(tag, deltas)
+    {:diff, puts, updates, deletes, state}
   end
 
-  def handle_catch_up(%__MODULE__{tag: tag} = state, from, {:state_transfer, tag, clock, state}) do
+  def handle_catch_up(%__MODULE__{tag: tag} = state, from, {:state_transfer, tag, clock, data}) do
+    %{clocks: clocks} = state
+
+    case tag do
+      :ignore -> {:insert, data, %{state | clocks: %{clocks | from => clock}}}
+    end
   end
 
-  def handle_broadcast(%__MODULE__{clocks: clocks, tag: tag}, ref, {tag, clock, data}) do
-    # handle_broadcast(tag, clocks, ref, clock, data)
+  # TODO: should we store somewhere we're catching up with the server?
+  # if so, then we should accumulate the broadcasts we can't handle, until we can.
+  def handle_broadcast(%__MODULE__{clocks: clocks, tag: tag} = state, ref, {tag, clock, delta}) do
+    case clocks do
+      %{^ref => old_clock} when old_clock + 1 == clock ->
+        state = %{state | clocks: %{clocks | ref => clock}}
+        {puts, updates, deletes} = handle_deltas(tag, [delta])
+        {:diff, puts, updates, deletes, state}
+
+      # We missed some broadcast, catch up with the node
+      %{^ref => old_clock} when clock > old_clock ->
+        {:catch_up, {clock, old_clock}, state}
+
+      # We were caught up with a newer clock than the current, ignore
+      # TODO: is that even possible?
+      %{^ref => old_clock} when clock < old_clock ->
+        {:ok, state}
+    end
   end
 
   def handle_broadcast(%__MODULE__{tag: local_tag}, ref, {remote_tag, _, _}) do
     {:error, {:different_tag, ref, local_tag, remote_tag}}
   end
 
-  def local_put(state, key, pid) do
-    event(state, key, pid, :put)
+  defp handle_deltas(:ignore, deltas) do
+    handler = &handle_ignore_delta/4
+    handle_deltas(deltas, [], [], [], handler)
+  end
+
+  defp handle_deltas([delta], inserts, updates, deletes, handler) do
+    handler.(delta, inserts, updates, deletes)
+  end
+
+  defp handle_deltas([delta | rest], inserts, updates, deletes, handler) do
+    {inserts, updates, deletes} = handler.(delta, inserts, updates, deletes)
+    handle_deltas(rest, inserts, updates, deletes, handler)
+  end
+
+  def local_put(state, key, pid, value) do
+    event(state, key, pid, {:put, value})
   end
 
   def local_delete(state, key, pid) do
     event(state, key, pid, :delete)
   end
 
-  def local_update(state, key, pid) do
-    event(state, key, pid, :update)
+  def local_update(state, key, pid, value, delta) do
+    event(state, key, pid, {:update, value, delta})
   end
 
   defp event(%__MODULE__{pending: pending, tag: tag} = state, key, pid, event) do
@@ -92,41 +136,50 @@ defmodule Firenest.ReplicatedState.Remote do
     %{state | pending: pending}
   end
 
-  def event_ignore(pending, key, pid, :put) do
-    Map.put(pending, {key, pid}, :put)
+  defp event_ignore(pending, key, pid, {:put, value}) do
+    Map.put(pending, {key, pid}, {:put, value})
   end
 
-  def event_ignore(pending, key, pid, :delete) do
+  defp event_ignore(pending, key, pid, :delete) do
     pending_key = {key, pid}
 
     case pending do
-      %{^pending_key => :put} -> Map.delete(pending, pending_key)
+      %{^pending_key => {:put, _}} -> Map.delete(pending, pending_key)
       %{} -> Map.put(pending, pending_key, :delete)
     end
   end
 
-  def event_ignore(pending, key, pid, :update) do
+  defp event_ignore(pending, key, pid, {:update, value, delta}) do
     pending_key = {key, pid}
 
     case pending do
-      %{^pending_key => :put} -> pending
-      %{} -> Map.put(pending, pending_key, :update)
+      %{^pending_key => {:put, _}} -> %{pending | pending_key => {:put, value}}
+      %{} -> Map.put(pending, pending_key, {:update, delta})
     end
   end
 
-  defp prepare_deltas(:ignore, pending, getter) do
-    Enum.map(pending, &prepare_ignore_delta(&1, getter))
+  defp prepare_deltas(:ignore, pending, prepare) do
+    prepare_ignore_deltas(pending, [], [], [], prepare)
+  end
+
+  defp prepare_ignore_deltas([], puts, updates, deletes, _prepare) do
+    {puts, updates, deletes}
+  end
+
+  defp prepare_ignore_deltas([{key, {:put, value}} | rest], puts, updates, deletes, prepare) do
+    prepare_ignore_deltas(rest, [{key, value} | puts], updates, deletes, prepare)
   end
 
-  defp prepare_ignore_delta({{key, pid}, :put}, getter) do
-    {:put, key, getter.(key, pid)}
+  defp prepare_ignore_deltas([{key, {:update, delta}} | rest], puts, deletes, updates, prepare) do
+    delta = prepare.(delta)
+    prepare_ignore_deltas(rest, puts, [{key, delta} | updates], deletes, prepare)
   end
 
-  defp prepare_ignore_delta({{key, pid}, :update}, getter) do
-    {:put, key, getter.(key, pid)}
+  defp prepare_ignore_deltas([{key, :delete} | rest], puts, deletes, updates, prepare) do
+    prepare_ignore_deltas(rest, puts, updates, [key | deletes], prepare)
   end
 
-  defp prepare_ignore_delta({{key, _pid}, :delete}, _getter) do
-    {:delete, key}
+  defp handle_ignore_delta({delta_puts, delta_updates, delta_deletes}, puts, updates, deletes) do
+    {delta_puts ++ puts, delta_updates ++ updates, delta_deletes ++ deletes}
   end
 end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index b8977b1..087a31b 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -34,7 +34,7 @@ defmodule Firenest.ReplicatedState.Server do
        handler: handler,
        remote: remote,
        broadcast_timer: nil,
-       broadcast_timeout: broadcast_timeout,
+       broadcast_timeout: broadcast_timeout
      }}
   end
 
@@ -156,7 +156,7 @@ defmodule Firenest.ReplicatedState.Server do
   # def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
   #   %{pending_events: events, clock: clock} = state
   #   clock = clock + 1
-  #   SyncedServer.remote_broadcast({:events, clock, events})
+  #   SyncedServer.remote_broadcast({:broadcast, clock, events})
   #   {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
   # end
 
@@ -193,64 +193,77 @@ defmodule Firenest.ReplicatedState.Server do
     end
   end
 
-  # @impl true
-  # def handle_remote({:catch_up_req, clock}, from, state) do
-  #   {mode, data} = catch_up_reply(state, clock)
-  #   SyncedServer.remote_send(from, {:catch_up, mode, data})
-  #   {:noreply, state}
-  # end
+  @impl true
+  def handle_remote({:catch_up_req, data}, from, state) do
+    %{remote: remote, store: store} = state
 
-  # def handle_remote({:catch_up, :state_transfer, {clock, transfer}}, from, state) do
-  #   state = handle_state_transfer(state, from, clock, transfer)
-  #   {:noreply, state}
-  # end
+    get_all_local = fn -> Store.list_local(store) end
+    reply = Remote.catch_up(remote, data, get_all_local)
+    SyncedServer.remote_send(from, {:catch_up_rep, reply})
+    {:noreply, state}
+  end
 
-  # def handle_remote({:events, remote_clock, events}, from, state) do
-  #   %{remote_clocks: remote_clocks} = state
-  #   local_clock = Map.fetch!(remote_clocks, from)
-
-  #   if remote_clock == local_clock + 1 do
-  #     remote_clocks = %{remote_clocks | from => remote_clock}
-  #     state = handle_events(state, from, events)
-  #     {:noreply, %{state | remote_clocks: remote_clocks}}
-  #   else
-  #     {:noreply, request_catch_up(state, from, local_clock)}
-  #   end
-  # end
+  def handle_remote({:catch_up_rep, data}, from, state) do
+    %{remote: remote, store: store, handler: handler} = state
 
-  # @impl true
-  # def handle_replica({:up, remote_clock}, remote_ref, state) do
-  #   %{remote_clocks: remote_clocks} = state
+    case Remote.handle_catch_up(remote, from, data) do
+      {:insert, data, remote} ->
+        store = Store.remote_update(store, from, data)
+        {:noreply, %{state | remote: remote, store: store}}
 
-  #   case remote_clocks do
-  #     %{^remote_ref => old_clock} when remote_clock > old_clock ->
-  #       # Reconnection, try to catch up
-  #       {:noreply, request_catch_up(state, remote_ref, old_clock)}
-
-  #     %{^remote_ref => old_clock} ->
-  #       # Reconnection, no remote state change, skip catch up
-  #       # Assert for sanity
-  #       true = old_clock == remote_clock
-  #       {:noreply, state}
-
-  #     %{} when remote_clock == 0 ->
-  #       # New node, no state, don't catch up
-  #       state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
-  #       {:noreply, state}
-
-  #     %{} ->
-  #       # New node, catch up
-  #       state = %{state | remote_clocks: Map.put(remote_clocks, remote_ref, 0)}
-  #       {:noreply, request_catch_up(state, remote_ref, 0)}
-  #   end
-  # end
+      {:diff, puts, updates, deletes, remote} ->
+        update_handler = &Handler.handle_remote_delta(handler, &1, &2)
+        store = Store.remote_diff(store, puts, updates, deletes, update_handler)
+        {:noreply, %{state | remote: remote, store: store}}
 
-  # def handle_replica(:down, remote_ref, state) do
-  #   %{values: values, remote_clocks: remote_clocks} = state
-  #   delete_ms = [{{:_, remote_ref, :_}, [], [true]}]
-  #   :ets.select_delete(values, delete_ms)
-  #   {:noreply, %{state | remote_clocks: Map.delete(remote_clocks, remote_ref)}}
-  # end
+      {:ok, remote} ->
+        {:noreply, %{state | remote: remote}}
+    end
+  end
+
+  def handle_remote({:broadcast, data}, from, state) do
+    %{remote: remote, store: store, hander: handler} = state
+
+    case Remote.handle_broadcast(remote, from, data) do
+      {:diff, puts, updates, deletes, remote} ->
+        update_handler = &Handler.handle_remote_delta(handler, &1, &2)
+        store = Store.remote_diff(store, puts, updates, deletes, update_handler)
+        {:noreply, %{state | remote: remote, store: store}}
+
+      {:catch_up, data, remote} ->
+        SyncedServer.remote_send(from, {:catch_up_req, data})
+        {:noreply, %{state | remote: remote}}
+
+      {:ok, remote} ->
+        {:noreply, %{state | remote: remote}}
+    end
+  end
+
+  @impl true
+  def handle_replica(change, remote_ref, state) do
+    %{remote: remote, store: store} = state
+
+    case remote_replica(change, remote_ref, remote) do
+      {:ok, remote} ->
+        {:noreply, %{state | remote: remote}}
+
+      {:delete, remote} ->
+        store = Store.remote_delete(store, remote_ref)
+        {:noreply, %{state | remote: remote, store: store}}
+
+      {:catch_up, data, remote} ->
+        SyncedServer.remote_send(remote_ref, {:catch_up_req, data})
+        {:noreply, %{state | remote: remote}}
+    end
+  end
+
+  defp remote_replica({:up, clock}, ref, remote) do
+    Remote.up(remote, ref, clock)
+  end
+
+  defp remote_replica(:down, ref, remote) do
+    Remote.down(remote, ref)
+  end
 
   defp link(:partition), do: true
   defp link(pid), do: Process.link(pid)
diff --git a/lib/firenest/replicated_state/store.ex b/lib/firenest/replicated_state/store.ex
index 30f5720..b6a94ad 100644
--- a/lib/firenest/replicated_state/store.ex
+++ b/lib/firenest/replicated_state/store.ex
@@ -1,6 +1,9 @@
 defmodule Firenest.ReplicatedState.Store do
   defstruct [:values, :pids]
 
+  # Common data exchange format:
+  # {{key, pid}, value}
+
   def new(name) do
     values = :ets.new(name, [:named_table, :protected, :ordered_set, read_concurrency: true])
     pids = :ets.new(__MODULE__.Pids, [:private, :duplicate_bag, keypos: 2])
@@ -10,8 +13,13 @@ defmodule Firenest.ReplicatedState.Store do
 
   def list(%__MODULE__{values: values}, key) do
     local = {{{key, :_}, :"$1", :_}, [], [:"$1"]}
-    # remote = {{{key, :_}, :_, :"$1"}, [], [:"$1"]}
-    :ets.select(values, [local])
+    remote = {{{key, :_}, :"$1"}, [], [:"$1"]}
+    :ets.select(values, [local, remote])
+  end
+
+  def list_local(%__MODULE__{values: values}) do
+    ms = [{{:"$1", :"$2", :_}, [], [{{:"$1", :"$2"}}]}]
+    :ets.select(values, ms)
   end
 
   def present?(%__MODULE__{values: values}, key, pid) do
@@ -76,6 +84,36 @@ defmodule Firenest.ReplicatedState.Store do
     state
   end
 
+  def remote_delete(%__MODULE__{values: values} = state, {node, _}) when is_atom(node) do
+    remote_delete_values(values, node)
+    state
+  end
+
+  def remote_update(%__MODULE__{values: values} = state, {node, _}, data) when is_atom(node) do
+    remote_delete_values(values, node)
+    :ets.insert(values, data)
+    state
+  end
+
+  def remote_diff(%__MODULE__{values: values} = state, puts, updates, deletes, update_handler) do
+    delete_ms = for key <- deletes, do: {{key, :_}, [], true}
+    puts = Enum.reduce(updates, puts, &prepare_update(values, update_handler, &1))
+    :ets.insert(values, puts)
+    :ets.select_delete(values, delete_ms)
+    state
+  end
+
+  defp remote_delete_values(values, node) do
+    ms = [{{{:_, :"$1"}, :_}, [{:"=:=", {:node, :"$1"}, {:const, node}}], [true]}]
+    :ets.select_delete(values, ms)
+  end
+
+  defp prepare_update(values, {key, delta}, update_handler) do
+    value = :ets.lookup_element(values, key, 2)
+    new_value = update_handler.(delta, value)
+    {key, new_value}
+  end
+
   if function_exported?(:ets, :whereis, 1) do
     defp ets_whereis(table), do: :ets.whereis(table)
   else

From 9352787b368538a4b04872ace05a4c5f894acd74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 12 Nov 2018 14:37:41 +0100
Subject: [PATCH 26/40] Properly handle broadcasts in remote server

---
 lib/firenest/replicated_state/handler.ex | 10 ++++
 lib/firenest/replicated_state/remote.ex  | 26 +++++++---
 lib/firenest/replicated_state/server.ex  | 62 +++++++++++++-----------
 3 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/lib/firenest/replicated_state/handler.ex b/lib/firenest/replicated_state/handler.ex
index 16d64e2..db06705 100644
--- a/lib/firenest/replicated_state/handler.ex
+++ b/lib/firenest/replicated_state/handler.ex
@@ -50,4 +50,14 @@ defmodule Firenest.ReplicatedState.Handler do
 
     mod.handle_remote_delta(delta, value, config)
   end
+
+  def prepare_remote_delta_fun(%__MODULE__{} = state) do
+    %{mod: mod, config: config} = state
+
+    if function_exported?(mod, :prepare_remote_delta, 2) do
+      &mod.prepare_remote_delta(&1, config)
+    else
+      &(&1)
+    end
+  end
 end
diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index 0f2508e..115937b 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -1,11 +1,11 @@
 defmodule Firenest.ReplicatedState.Remote do
-  defstruct pending: %{}, clocks: %{}, clock: 0, tag: nil, deltas: %{}
+  defstruct pending: %{}, clocks: %{}, clock: 0, tag: nil, deltas: %{}, broadcast: nil
 
   # TODO: some protocol for requesting more of a state, even from other nodes
   # on first up, so we don't need to go to each node separately.
 
-  def new(:ignore) do
-    %__MODULE__{tag: :ignore}
+  def new(:ignore, broadcast) do
+    %__MODULE__{tag: :ignore, broadcast: broadcast}
   end
 
   def clock(%__MODULE__{clock: clock}), do: clock
@@ -55,9 +55,10 @@ defmodule Firenest.ReplicatedState.Remote do
     end
   end
 
-  def broadcast(%__MODULE__{pending: pending, clock: clock, tag: tag} = state, prepare_delta) do
+  def broadcast(%__MODULE__{} = state, prepare_delta) do
+    %{pending: pending, clock: clock, tag: tag, broadcast: {:scheduled, broadcast}} = state
     deltas = prepare_deltas(tag, pending, prepare_delta)
-    new_state = %{state | pending: %{}, clock: clock + 1}
+    new_state = %{state | pending: %{}, clock: clock + 1, broadcast: broadcast}
     {{tag, clock, deltas}, new_state}
   end
 
@@ -127,13 +128,15 @@ defmodule Firenest.ReplicatedState.Remote do
     event(state, key, pid, {:update, value, delta})
   end
 
-  defp event(%__MODULE__{pending: pending, tag: tag} = state, key, pid, event) do
+  defp event(%__MODULE__{tag: tag} = state, key, pid, event) do
+    %{pending: pending, broadcast: broadcast} = state
+
     pending =
       case tag do
         :ignore -> event_ignore(pending, key, pid, event)
       end
 
-    %{state | pending: pending}
+    %{state | pending: pending, broadcast: broadcast(broadcast)}
   end
 
   defp event_ignore(pending, key, pid, {:put, value}) do
@@ -182,4 +185,13 @@ defmodule Firenest.ReplicatedState.Remote do
   defp handle_ignore_delta({delta_puts, delta_updates, delta_deletes}, puts, updates, deletes) do
     {delta_puts ++ puts, delta_updates ++ updates, delta_deletes ++ deletes}
   end
+
+  defp broadcast({:scheduled, fun}) do
+    {:scheduled, fun}
+  end
+
+  defp broadcast(fun) do
+    fun.()
+    {:scheduled, fun}
+  end
 end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 087a31b..720e970 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -20,9 +20,10 @@ defmodule Firenest.ReplicatedState.Server do
 
     store = Store.new(name)
 
-    broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
     remote_changes = Keyword.get(opts, :remote_changes, :ignore)
-    remote = Remote.new(remote_changes)
+    broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
+    broadcast_fun = fn -> Process.send_after(self(), :broadcast_timeout, broadcast_timeout) end
+    remote = Remote.new(remote_changes, broadcast_fun)
 
     delayed_fun = &Process.send_after(self(), {:update, &1, &2, &3}, &4)
     # The `mode` should be read from init instead of options
@@ -33,8 +34,6 @@ defmodule Firenest.ReplicatedState.Server do
        store: store,
        handler: handler,
        remote: remote,
-       broadcast_timer: nil,
-       broadcast_timeout: broadcast_timeout
      }}
   end
 
@@ -43,20 +42,22 @@ defmodule Firenest.ReplicatedState.Server do
 
   @impl true
   def handle_call({:put, key, pid, arg}, _from, state) do
-    %{store: store, handler: handler} = state
+    %{store: store, handler: handler, remote: remote} = state
 
     link(pid)
 
     unless Store.present?(store, key, pid) do
       case Handler.local_put(handler, arg, key, pid) do
         {:put, value, delta, handler} ->
+          remote = Remote.local_put(remote, key, pid, value)
           store = Store.local_put(store, key, pid, value, delta)
-          {:reply, :ok, %{state | store: store, handler: handler}}
+          {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
 
         {:delete, value, _delta, handler} ->
-          # TODO: this delta has to propagate remotely before delete
+          remote = Remote.local_put(remote, key, pid, value)
+          remote = Remote.local_delete(remote, key, pid)
           handler = Handler.local_delete(handler, [value])
-          {:reply, :ok, %{state | handler: handler}}
+          {:reply, :ok, %{state | handler: handler, remote: remote}}
       end
     else
       {:reply, {:error, :already_present}, state}
@@ -64,29 +65,30 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:update, key, pid, arg}, _from, state) do
-    %{store: store, handler: handler} = state
+    %{store: store, handler: handler, remote: remote} = state
 
     case Store.fetch(store, key, pid) do
       {:ok, value, delta} ->
         case Handler.local_update(handler, arg, key, pid, delta, value) do
           {:put, value, delta, handler} ->
+            remote = Remote.local_update(remote, key, pid, value, delta)
             store = Store.local_update(store, key, pid, value, delta)
-            {:reply, :ok, %{state | store: store, handler: handler}}
+            {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
 
-          {:delete, value, _delta, handler} ->
-            # TODO: this delta has to propagate remotely before delete
+          {:delete, value, delta, handler} ->
+            remote = Remote.local_update(remote, key, pid, value, delta)
             case Store.local_delete(store, key, pid) do
               # The value returned from update is fresher
               {:ok, _value, store} ->
-                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+                remote = Remote.local_delete(remote, key, pid)
                 handler = Handler.local_delete(handler, [value])
-                {:reply, :ok, %{state | store: store, handler: handler}}
+                {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
 
               {:last_member, _value, store} ->
                 unlink_flush(pid)
+                remote = Remote.local_delete(remote, key, pid)
                 handler = Handler.local_delete(handler, [value])
-                # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-                {:reply, :ok, %{state | store: store, handler: handler}}
+                {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
             end
         end
 
@@ -96,19 +98,19 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:delete, key, pid}, _from, state) do
-    %{store: store, handler: handler} = state
+    %{store: store, handler: handler, remote: remote} = state
 
     case Store.local_delete(store, key, pid) do
       {:ok, value, store} ->
-        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
+        remote = Remote.local_delete(remote, key, pid)
         handler = Handler.local_delete(handler, [value])
-        {:reply, :ok, %{state | store: store, handler: handler}}
+        {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
 
       {:last_member, value, store} ->
         unlink_flush(pid)
+        remote = Remote.local_delete(remote, key, pid)
         handler = Handler.local_delete(handler, [value])
-        # state = schedule_broadcast_events(state, [{:leave, key, pid}])
-        {:reply, :ok, %{state | store: store, handler: handler}}
+        {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
 
       {:error, store} ->
         {:reply, {:error, :not_present}, %{state | store: store}}
@@ -122,7 +124,7 @@ defmodule Firenest.ReplicatedState.Server do
       {:ok, deletes, store} ->
         unlink_flush(pid)
         handler = Handler.local_delete(handler, deletes)
-        # state = schedule_broadcast_events(state, leaves)
+        # TODO: how do we handle remote in here?
         {:reply, :ok, %{state | store: store, handler: handler}}
 
       {:error, store} ->
@@ -145,7 +147,7 @@ defmodule Firenest.ReplicatedState.Server do
     case Store.local_delete(store, pid) do
       {:ok, leaves, store} ->
         handler = Handler.local_delete(handler, leaves)
-        # state = schedule_broadcast_events(state, leaves)
+        # TODO: how do we handle remote in here?
         {:noreply, %{state | store: store, handler: handler}}
 
       {:error, store} ->
@@ -153,12 +155,14 @@ defmodule Firenest.ReplicatedState.Server do
     end
   end
 
-  # def handle_info({:timeout, timer, :broadcast}, %{broadcast_timer: timer} = state) do
-  #   %{pending_events: events, clock: clock} = state
-  #   clock = clock + 1
-  #   SyncedServer.remote_broadcast({:broadcast, clock, events})
-  #   {:noreply, %{state | clock: clock, pending_events: [], broadcast_timer: nil}}
-  # end
+  def handle_info(:broadcast_timeout, state) do
+    %{remote: remote, handler: handler} = state
+
+    prepare_delta = Handler.prepare_remote_delta_fun(handler)
+    {data, remote} = Remote.broadcast(remote, prepare_delta)
+    SyncedServer.remote_broadcast({:broadcast, data})
+    {:noreply, %{state | remote: remote}}
+  end
 
   def handle_info({:update, key, pid, arg}, state) do
     %{store: store, handler: handler} = state

From 055b900f3c30eb7246778c4d8589aa212529facd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Mon, 12 Nov 2018 14:39:41 +0100
Subject: [PATCH 27/40] Fix tests

---
 lib/firenest/replicated_state/remote.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index 115937b..837dfb2 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -162,7 +162,7 @@ defmodule Firenest.ReplicatedState.Remote do
   end
 
   defp prepare_deltas(:ignore, pending, prepare) do
-    prepare_ignore_deltas(pending, [], [], [], prepare)
+    prepare_ignore_deltas(Map.to_list(pending), [], [], [], prepare)
   end
 
   defp prepare_ignore_deltas([], puts, updates, deletes, _prepare) do

From b87a6ffc5029f352e6bf7428cc6363c23fe40b2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Tue, 13 Nov 2018 23:01:19 +0100
Subject: [PATCH 28/40] Formatter and remove old code

---
 lib/firenest/replicated_state/server.ex | 58 +------------------------
 1 file changed, 2 insertions(+), 56 deletions(-)

diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 720e970..821c694 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -33,7 +33,7 @@ defmodule Firenest.ReplicatedState.Server do
      %{
        store: store,
        handler: handler,
-       remote: remote,
+       remote: remote
      }}
   end
 
@@ -77,6 +77,7 @@ defmodule Firenest.ReplicatedState.Server do
 
           {:delete, value, delta, handler} ->
             remote = Remote.local_update(remote, key, pid, value, delta)
+
             case Store.local_delete(store, key, pid) do
               # The value returned from update is fresher
               {:ok, _value, store} ->
@@ -283,59 +284,4 @@ defmodule Firenest.ReplicatedState.Server do
       0 -> true
     end
   end
-
-  # defp schedule_broadcast_events(%{broadcast_timer: nil} = state, new_events) do
-  #   %{broadcast_timeout: timeout, pending_events: events} = state
-  #   timer = :erlang.start_timer(timeout, self(), :broadcast)
-  #   %{state | broadcast_timer: timer, pending_events: new_events ++ events}
-  # end
-
-  # defp schedule_broadcast_events(%{} = state, new_events) do
-  #   %{pending_events: events} = state
-  #   %{state | pending_events: new_events ++ events}
-  # end
-
-  # defp request_catch_up(state, remote_ref, clock) do
-  #   SyncedServer.remote_send(remote_ref, {:catch_up_req, clock})
-  #   state
-  # end
-
-  # defp handle_events(%{values: values} = state, from, events) do
-  #   {joins, leaves} =
-  #     Enum.reduce(events, {[], []}, fn
-  #       {:leave, key, pid}, {joins, leaves} ->
-  #         leave = {{{key, pid}, from, :_}, [], [true]}
-  #         {joins, [leave | leaves]}
-
-  #       {:replace, key, pid, value}, {joins, leaves} ->
-  #         join = {{key, pid}, from, value}
-  #         {[join | joins], leaves}
-
-  #       {:join, key, pid, value}, {joins, leaves} ->
-  #         join = {{key, pid}, from, value}
-  #         {[join | joins], leaves}
-  #     end)
-
-  #   :ets.insert(values, joins)
-  #   :ets.select_delete(values, leaves)
-  #   state
-  # end
-
-  # # TODO: detect leaves
-  # # Is there a better way than to clean up and re-insert?
-  # # This can be problematic for dirty reads!
-  # defp handle_state_transfer(%{values: values} = state, from, clock, transfer) do
-  #   %{remote_clocks: remote_clocks} = state
-  #   delete_ms = [{{:_, from, :_}, [], [true]}]
-  #   inserts = for {ets_key, value} <- transfer, do: {ets_key, from, value}
-  #   :ets.select_delete(values, delete_ms)
-  #   :ets.insert(values, inserts)
-  #   %{state | remote_clocks: %{remote_clocks | from => clock}}
-  # end
-
-  # # TODO: handle catch-up with events
-  # defp catch_up_reply(%{values: values}, clock) do
-  #   local_ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}]
-  #   {:state_transfer, {clock, :ets.select(values, local_ms)}}
-  # end
 end

From 906e6d94e77a3270488563385e039265036292a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 14 Nov 2018 12:16:42 +0100
Subject: [PATCH 29/40] Add ReplicatedState.Store tests

---
 lib/firenest/replicated_state/server.ex       |  12 +-
 lib/firenest/replicated_state/store.ex        |   8 +-
 test/firenest/replicated_state/store_test.exs | 153 ++++++++++++++++++
 test/test_helper.exs                          |  21 +--
 4 files changed, 174 insertions(+), 20 deletions(-)
 create mode 100644 test/firenest/replicated_state/store_test.exs

diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 821c694..53e6a96 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -113,8 +113,8 @@ defmodule Firenest.ReplicatedState.Server do
         handler = Handler.local_delete(handler, [value])
         {:reply, :ok, %{state | store: store, handler: handler, remote: remote}}
 
-      {:error, store} ->
-        {:reply, {:error, :not_present}, %{state | store: store}}
+      :error ->
+        {:reply, {:error, :not_present}, state}
     end
   end
 
@@ -128,8 +128,8 @@ defmodule Firenest.ReplicatedState.Server do
         # TODO: how do we handle remote in here?
         {:reply, :ok, %{state | store: store, handler: handler}}
 
-      {:error, store} ->
-        {:reply, {:error, :not_member}, %{state | store: store}}
+      :error ->
+        {:reply, {:error, :not_member}, state}
     end
   end
 
@@ -151,8 +151,8 @@ defmodule Firenest.ReplicatedState.Server do
         # TODO: how do we handle remote in here?
         {:noreply, %{state | store: store, handler: handler}}
 
-      {:error, store} ->
-        {:stop, reason, %{state | store: store}}
+      :error ->
+        {:stop, reason, state}
     end
   end
 
diff --git a/lib/firenest/replicated_state/store.ex b/lib/firenest/replicated_state/store.ex
index b6a94ad..74114d0 100644
--- a/lib/firenest/replicated_state/store.ex
+++ b/lib/firenest/replicated_state/store.ex
@@ -50,7 +50,7 @@ defmodule Firenest.ReplicatedState.Store do
 
     case :ets.select_delete(pids, ms) do
       0 ->
-        {:error, state}
+        :error
 
       1 ->
         [{_, value, _}] = :ets.take(values, ets_key)
@@ -66,7 +66,7 @@ defmodule Firenest.ReplicatedState.Store do
   def local_delete(%__MODULE__{values: values, pids: pids} = state, pid) do
     case :ets.take(pids, pid) do
       [] ->
-        {:error, state}
+        :error
 
       list ->
         delete_ms = for ets_key <- list, do: {{ets_key, :_, :_}, [], [true]}
@@ -96,8 +96,8 @@ defmodule Firenest.ReplicatedState.Store do
   end
 
   def remote_diff(%__MODULE__{values: values} = state, puts, updates, deletes, update_handler) do
-    delete_ms = for key <- deletes, do: {{key, :_}, [], true}
-    puts = Enum.reduce(updates, puts, &prepare_update(values, update_handler, &1))
+    delete_ms = for key <- deletes, do: {{key, :_}, [], [true]}
+    puts = Enum.reduce(updates, puts, &[prepare_update(values, &1, update_handler) | &2])
     :ets.insert(values, puts)
     :ets.select_delete(values, delete_ms)
     state
diff --git a/test/firenest/replicated_state/store_test.exs b/test/firenest/replicated_state/store_test.exs
new file mode 100644
index 0000000..e3c4cc6
--- /dev/null
+++ b/test/firenest/replicated_state/store_test.exs
@@ -0,0 +1,153 @@
+defmodule Firenest.ReplicatedState.StoreTest do
+  use ExUnit.Case, async: true
+
+  alias Firenest.ReplicatedState.Store
+
+  setup %{test: test} do
+    store = Store.new(test)
+    other = spawn_link(fn -> Process.sleep(:infinity) end)
+    [store: store, other: other]
+  end
+
+  test "empty store", %{store: store} do
+    assert Store.list(store, :a) == []
+    assert Store.list_local(store) == []
+    refute Store.present?(store, :a, self())
+  end
+
+  test "list with entries", %{store: store, other: other} do
+    store =
+      store
+      |> Store.local_put(:a, self(), 1, 1)
+      |> Store.local_put(:b, other, 2, 2)
+
+    assert Store.list(store, :a) == [1]
+    assert Store.list(store, :b) == [2]
+    assert Store.list(store, :c) == []
+    assert Store.list_local(store) == [{{:a, self()}, 1}, {{:b, other}, 2}]
+  end
+
+  test "present?/3", %{store: store, other: other} do
+    store =
+      store
+      |> Store.local_put(:a, self(), 1, 1)
+      |> Store.local_put(:b, other, 2, 2)
+
+    assert Store.present?(store, :a, self())
+    assert Store.present?(store, :b, other)
+    refute Store.present?(store, :c, self())
+    refute Store.present?(store, :a, other)
+  end
+
+  test "fetch/3", %{store: store, other: other} do
+    store =
+      store
+      |> Store.local_put(:a, self(), 1, 1)
+      |> Store.local_put(:b, other, 2, 2)
+
+    assert Store.fetch(store, :a, self()) == {:ok, 1, 1}
+    assert Store.fetch(store, :b, other) == {:ok, 2, 2}
+    assert Store.fetch(store, :c, self()) == :error
+    assert Store.fetch(store, :a, other) == :error
+  end
+
+  test "local_update/5", %{store: store, other: other} do
+    store =
+      store
+      |> Store.local_put(:a, self(), 1, 1)
+      |> Store.local_put(:b, other, 2, 2)
+      |> Store.local_update(:a, self(), 3, 3)
+      |> Store.local_update(:b, other, 4, 4)
+
+    assert Store.fetch(store, :a, self()) == {:ok, 3, 3}
+    assert Store.fetch(store, :b, other) == {:ok, 4, 4}
+  end
+
+  test "local_delete/2", %{store: store, other: other} do
+    store =
+      store
+      |> Store.local_put(:a, self(), 1, 1)
+      |> Store.local_put(:b, other, 2, 2)
+
+    another = spawn_link(fn -> Process.sleep(:infinity) end)
+
+    assert Store.local_delete(store, another) == :error
+
+    assert {:ok, [1], store} = Store.local_delete(store, self())
+    refute Store.present?(store, :a, self())
+    assert Store.present?(store, :b, other)
+  end
+
+  test "local_delete/3", %{store: store, other: other} do
+    store =
+      store
+      |> Store.local_put(:a, self(), 1, 1)
+      |> Store.local_put(:b, other, 2, 2)
+      |> Store.local_put(:c, self(), 3, 3)
+
+    another = spawn_link(fn -> Process.sleep(:infinity) end)
+
+    assert Store.local_delete(store, :a, another) == :error
+
+    assert {:ok, 1, store} = Store.local_delete(store, :a, self())
+    assert {:last_member, 3, store} = Store.local_delete(store, :c, self())
+    refute Store.present?(store, :a, self())
+    refute Store.present?(store, :c, self())
+    assert Store.present?(store, :b, other)
+  end
+
+  test "remote_update/3", %{store: store} do
+    node_ref = {:node1, 1}
+    data = [{{:a, remote_pid(:node1, 1)}, 1}]
+    store = Store.remote_update(store, node_ref, data)
+
+    assert Store.list(store, :a) == [1]
+
+    store = Store.local_put(store, :a, self(), 2, 2)
+
+    assert Store.list(store, :a) == [1, 2]
+
+    new_data = [{{:b, remote_pid(:node1, 1)}, 3}]
+    store = Store.remote_update(store, node_ref, new_data)
+
+    assert Store.list(store, :a) == [2]
+  end
+
+  test "remote_delete/2", %{store: store} do
+    data1 = [{{:a, remote_pid(:node1, 1)}, 1}]
+    data2 = [{{:a, remote_pid(:node2, 1)}, 2}]
+
+    store =
+      store
+      |> Store.remote_update({:node1, 1}, data1)
+      |> Store.remote_update({:node2, 1}, data2)
+      |> Store.remote_delete({:node1, 1})
+
+    assert Store.list(store, :a) == [2]
+  end
+
+  test "remote_diff/5", %{store: store} do
+    parent = self()
+
+    update_handler = fn delta, value ->
+      send(parent, {:update, delta, value})
+      delta + value
+    end
+
+    data = [{{:a, remote_pid(:node1, 1)}, 1}, {{:a, remote_pid(:node1, 2)}, 2}]
+    store = Store.remote_update(store, {:node1, 1}, data)
+
+    puts = [{{:a, remote_pid(:node1, 3)}, 3}]
+    updates = [{{:a, remote_pid(:node1, 2)}, 4}]
+    deletes = [{:a, remote_pid(:node1, 1)}]
+    store = Store.remote_diff(store, puts, updates, deletes, update_handler)
+
+    assert Store.list(store, :a) == [6, 3]
+    assert_received {:update, 4, 2}
+  end
+
+  defp remote_pid(node, num) do
+    <<131, 100, node::binary>> = :erlang.term_to_binary(node)
+    :erlang.binary_to_term(<<131, 103, 100, node::binary, num::8*4, 0::8*4, 0::8*1>>)
+  end
+end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index abd54de..ebf8e40 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -4,15 +4,16 @@ nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
 Firenest.Test.start_boot_server(hd(nodes))
 Firenest.Test.start_firenest([hd(nodes)], adapter: Firenest.Topology.Erlang)
 
+parent = self()
+
 # Start other nodes async, so we can start running tests that don't need them right away
-pid =
-  spawn_link(fn ->
-    receive do: (:continue -> :ok)
-    Firenest.Test.spawn_nodes(tl(nodes))
-    Firenest.Test.start_firenest(tl(nodes), adapter: Firenest.Topology.Erlang)
-    Process.unregister(:firenest_topology_setup)
-    Process.sleep(:infinity)
-  end)
+spawn_link(fn ->
+  Process.register(self(), :firenest_topology_setup)
+  send(parent, :continue)
+  Firenest.Test.spawn_nodes(tl(nodes))
+  Firenest.Test.start_firenest(tl(nodes), adapter: Firenest.Topology.Erlang)
+  Process.unregister(:firenest_topology_setup)
+  Process.sleep(:infinity)
+end)
 
-Process.register(pid, :firenest_topology_setup)
-send(pid, :continue)
+receive do: (:continue -> :ok)

From 451ef3e01ce6fb67618a14e02206ff7a936344a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Thu, 15 Nov 2018 15:27:06 +0100
Subject: [PATCH 30/40] Move remote changes to handler return

---
 lib/firenest/replicated_state/handler.ex |  9 ++++++---
 lib/firenest/replicated_state/server.ex  | 11 ++++++-----
 lib/firenest/replicated_state/store.ex   |  2 ++
 test/support/eval_state.ex               |  2 +-
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/firenest/replicated_state/handler.ex b/lib/firenest/replicated_state/handler.ex
index db06705..7c3798c 100644
--- a/lib/firenest/replicated_state/handler.ex
+++ b/lib/firenest/replicated_state/handler.ex
@@ -1,9 +1,12 @@
 defmodule Firenest.ReplicatedState.Handler do
+  @moduledoc false
+
   defstruct mod: nil, init_delta: nil, config: nil, delayed_fun: nil
 
   def new(mod, mod_opts, delayed_fun) do
-    {init_delta, config} = mod.init(mod_opts)
-    %__MODULE__{mod: mod, init_delta: init_delta, config: config, delayed_fun: delayed_fun}
+    {delta, config, opts} = mod.init(mod_opts)
+    state = %__MODULE__{mod: mod, init_delta: delta, config: config, delayed_fun: delayed_fun}
+    {state, opts}
   end
 
   def local_put(%__MODULE__{} = state, arg, key, pid) do
@@ -57,7 +60,7 @@ defmodule Firenest.ReplicatedState.Handler do
     if function_exported?(mod, :prepare_remote_delta, 2) do
       &mod.prepare_remote_delta(&1, config)
     else
-      &(&1)
+      & &1
     end
   end
 end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 53e6a96..faf7a1c 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -20,14 +20,15 @@ defmodule Firenest.ReplicatedState.Server do
 
     store = Store.new(name)
 
-    remote_changes = Keyword.get(opts, :remote_changes, :ignore)
+    delayed_fun = &Process.send_after(self(), {:update, &1, &2, &3}, &4)
+    {handler, server_opts} = Handler.new(handler, opts, delayed_fun)
+
     broadcast_timeout = Keyword.get(opts, :broadcast_timeout, 50)
     broadcast_fun = fn -> Process.send_after(self(), :broadcast_timeout, broadcast_timeout) end
-    remote = Remote.new(remote_changes, broadcast_fun)
 
-    delayed_fun = &Process.send_after(self(), {:update, &1, &2, &3}, &4)
-    # The `mode` should be read from init instead of options
-    handler = Handler.new(handler, opts, delayed_fun)
+    remote_changes = Keyword.get(server_opts, :remote_changes, :ignore)
+    max_deltas = Keyword.get(opts, :max_remote_deltas, 5)
+    remote = Remote.new(remote_changes, broadcast_fun, max_deltas)
 
     {:ok,
      %{
diff --git a/lib/firenest/replicated_state/store.ex b/lib/firenest/replicated_state/store.ex
index 74114d0..132ede8 100644
--- a/lib/firenest/replicated_state/store.ex
+++ b/lib/firenest/replicated_state/store.ex
@@ -1,4 +1,6 @@
 defmodule Firenest.ReplicatedState.Store do
+  @moduledoc false
+
   defstruct [:values, :pids]
 
   # Common data exchange format:
diff --git a/test/support/eval_state.ex b/test/support/eval_state.ex
index b49fe26..cf11d07 100644
--- a/test/support/eval_state.ex
+++ b/test/support/eval_state.ex
@@ -3,7 +3,7 @@ defmodule Firenest.Test.EvalState do
 
   @impl true
   def init(opts) do
-    {0, opts}
+    {0, opts, opts}
   end
 
   @impl true

From c75d98aca33307602cf35f6f4d9292d07946a58b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Thu, 15 Nov 2018 15:27:45 +0100
Subject: [PATCH 31/40] Poor man's bounded queue for last deltas in remote

---
 lib/firenest/replicated_state/remote.ex | 50 ++++++++++++++++++-------
 1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index 837dfb2..af0451b 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -1,11 +1,17 @@
 defmodule Firenest.ReplicatedState.Remote do
-  defstruct pending: %{}, clocks: %{}, clock: 0, tag: nil, deltas: %{}, broadcast: nil
+  @moduledoc false
+
+  defstruct pending: %{}, clocks: %{}, clock: 0, tag: nil, deltas: nil, broadcast: nil
 
   # TODO: some protocol for requesting more of a state, even from other nodes
   # on first up, so we don't need to go to each node separately.
 
-  def new(:ignore, broadcast) do
-    %__MODULE__{tag: :ignore, broadcast: broadcast}
+  import Record
+  defrecord :deltas, max: nil, lowest: 0, store: %{}
+
+  def new(:ignore, broadcast, max_deltas)
+      when is_function(broadcast, 0) and is_integer(max_deltas) do
+    %__MODULE__{tag: :ignore, broadcast: broadcast, deltas: deltas(max: max_deltas)}
   end
 
   def clock(%__MODULE__{clock: clock}), do: clock
@@ -15,7 +21,7 @@ defmodule Firenest.ReplicatedState.Remote do
     case clocks do
       # Reconnection, try to catch up
       %{^ref => old_clock} when clock > old_clock ->
-        {:catch_up, {clock, old_clock}, state}
+        {:catch_up, old_clock, state}
 
       # Reconnection, no remote changes
       %{^ref => old_clock} ->
@@ -29,7 +35,7 @@ defmodule Firenest.ReplicatedState.Remote do
 
       # New node, catch up
       %{} ->
-        {:catch_up, {clock, 0}, state}
+        {:catch_up, 0, state}
     end
   end
 
@@ -44,8 +50,8 @@ defmodule Firenest.ReplicatedState.Remote do
     {:delete, ref, %{state | clocks: clocks}}
   end
 
-  def catch_up(%__MODULE__{clock: current} = state, {clock, old_clock}, state_getter)
-      when old_clock < clock and clock <= current do
+  def catch_up(%__MODULE__{clock: current} = state, old_clock, state_getter)
+      when old_clock < current do
     %{deltas: deltas, tag: tag} = state
 
     if Map.has_key?(deltas, old_clock) do
@@ -56,10 +62,16 @@ defmodule Firenest.ReplicatedState.Remote do
   end
 
   def broadcast(%__MODULE__{} = state, prepare_delta) do
-    %{pending: pending, clock: clock, tag: tag, broadcast: {:scheduled, broadcast}} = state
-    deltas = prepare_deltas(tag, pending, prepare_delta)
-    new_state = %{state | pending: %{}, clock: clock + 1, broadcast: broadcast}
-    {{tag, clock, deltas}, new_state}
+    %{pending: pending, clock: clock, tag: tag, broadcast: broadcast, deltas: deltas} = state
+    {:scheduled, broadcast} = broadcast
+
+    new_deltas = prepare_deltas(tag, pending, prepare_delta)
+
+    new_clock = clock + 1
+    deltas = store_deltas(deltas, new_clock, new_deltas)
+    new_state = %{state | pending: %{}, clock: new_clock, broadcast: broadcast, deltas: deltas}
+
+    {{tag, clock, new_deltas}, new_state}
   end
 
   def handle_catch_up(%__MODULE__{tag: tag} = state, ref, {:deltas, tag, clock, deltas}) do
@@ -89,7 +101,7 @@ defmodule Firenest.ReplicatedState.Remote do
 
       # We missed some broadcast, catch up with the node
       %{^ref => old_clock} when clock > old_clock ->
-        {:catch_up, {clock, old_clock}, state}
+        {:catch_up, old_clock, state}
 
       # We were caught up with a newer clock than the current, ignore
       # TODO: is that even possible?
@@ -173,12 +185,12 @@ defmodule Firenest.ReplicatedState.Remote do
     prepare_ignore_deltas(rest, [{key, value} | puts], updates, deletes, prepare)
   end
 
-  defp prepare_ignore_deltas([{key, {:update, delta}} | rest], puts, deletes, updates, prepare) do
+  defp prepare_ignore_deltas([{key, {:update, delta}} | rest], puts, updates, deletes, prepare) do
     delta = prepare.(delta)
     prepare_ignore_deltas(rest, puts, [{key, delta} | updates], deletes, prepare)
   end
 
-  defp prepare_ignore_deltas([{key, :delete} | rest], puts, deletes, updates, prepare) do
+  defp prepare_ignore_deltas([{key, :delete} | rest], puts, updates, deletes, prepare) do
     prepare_ignore_deltas(rest, puts, updates, [key | deletes], prepare)
   end
 
@@ -194,4 +206,14 @@ defmodule Firenest.ReplicatedState.Remote do
     fun.()
     {:scheduled, fun}
   end
+
+  defp store_deltas(deltas(max: max, lowest: lowest, store: store), clock, new_delta) do
+    store = Map.put(store, clock, new_delta)
+
+    if map_size(store) > max do
+      deltas(max: max, lowest: lowest + 1, store: Map.delete(store, lowest))
+    else
+      deltas(max: max, lowest: lowest, store: store)
+    end
+  end
 end

From 61792d4f6fc48329946a726528ac6173672650f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 16 Nov 2018 15:28:04 +0100
Subject: [PATCH 32/40] Some remote unit tests

---
 .../firenest/replicated_state/remote_test.exs | 64 +++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 test/firenest/replicated_state/remote_test.exs

diff --git a/test/firenest/replicated_state/remote_test.exs b/test/firenest/replicated_state/remote_test.exs
new file mode 100644
index 0000000..a3dfcbd
--- /dev/null
+++ b/test/firenest/replicated_state/remote_test.exs
@@ -0,0 +1,64 @@
+defmodule Firenest.ReplicatedState.RemoteTest do
+  use ExUnit.Case, async: true
+
+  alias Firenest.ReplicatedState.Remote
+
+  @moduletag remote_changes: :ignore
+
+  setup %{remote_changes: changes} do
+    parent = self()
+    broadcast = fn -> send(parent, :broadcast) end
+    remote = Remote.new(changes, broadcast, 1)
+    [remote: remote]
+  end
+
+  test "up and down", %{remote: remote} do
+    assert {:ok, remote} = Remote.up(remote, :node1, 0)
+    assert {:catch_up, 0, remote} = Remote.up(remote, :node2, 5)
+    assert {:delete, :node1, _remote} = Remote.down(remote, :node1)
+  end
+
+  describe "remote_changes: :ignore" do
+    @describetag remote_changes: :ignore
+
+    test "local_put/4", %{remote: remote} do
+      remote =
+        remote
+        |> Remote.local_put(:a, self(), 1)
+        |> Remote.local_put(:b, self(), 2)
+
+      {data, _remote} = Remote.broadcast(remote, &identity/1)
+      assert {:ignore, 0, {puts, _updates = [], _deletes = []}} = data
+      assert [{{:b, self()}, 2}, {{:a, self()}, 1}] == puts
+    end
+
+    test "local_update/5", %{remote: remote} do
+      remote =
+        remote
+        |> Remote.local_update(:a, self(), 1, 2)
+        |> Remote.local_put(:b, self(), 3)
+        |> Remote.local_update(:b, self(), 4, 5)
+
+      {data, _remote} = Remote.broadcast(remote, &identity/1)
+      assert {:ignore, 0, {puts, updates, _deletes = []}} = data
+      assert [{{:b, self()}, 4}] == puts
+      assert [{{:a, self()}, 2}] == updates
+    end
+
+    test "local_delete/3", %{remote: remote} do
+      remote =
+        remote
+        |> Remote.local_update(:a, self(), 1, 2)
+        |> Remote.local_put(:b, self(), 3)
+        |> Remote.local_delete(:a, self())
+        |> Remote.local_delete(:b, self())
+        |> Remote.local_delete(:c, self())
+
+      {data, _remote} = Remote.broadcast(remote, &identity/1)
+      assert {:ignore, 0, {_puts = [], _updates = [], deletes}} = data
+      assert [{:c, self()}, {:a, self()}] == deletes
+    end
+  end
+
+  defp identity(x), do: x
+end

From 32707730ac0f4c38ba14c3ce598af444d1710bb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 21 Nov 2018 11:28:21 +0100
Subject: [PATCH 33/40] Update replicated state docs

---
 lib/firenest/replicated_state.ex     | 13 ++++++++-----
 test/firenest/synced_server_test.exs |  4 ++--
 test/shared/topology_test.exs        |  2 +-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 4a77cc2..77b39f5 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -49,6 +49,10 @@ defmodule Firenest.ReplicatedState do
       callbacks.
     * a list of server_opt` settings configuring the behaviour of the server
 
+        * `:remote_changes` - specifies behaviour of the `c:observe_remote_changes/2`
+          callback and can be `:observe_full | :observe_collapsed | :ignore`,
+          defaults to `:ignore`;
+
   """
   @callback init(opts :: keyword()) ::
               {initial_delta :: local_delta(), callback_config(), [server_opt]}
@@ -182,11 +186,10 @@ defmodule Firenest.ReplicatedState do
     * `:name` - name for the process, required;
     * `:topology` - name of the supporting topology, required;
     * `:partitions` - number of partitions, defaults to 1;
-    * `:broadcast_timeout` - delay of broadcasting local events to other nodes,
-      defaults to 50 ms;
-    * `:remote_changes` - specifies behaviour of the `c:observe_remote_changes/2`
-      callback and can be `:observe_full | :observe_collapsed | :ignore`,
-      defaults to `:ignore`;
+    * `:broadcast_timeout` - delay (in milliseconds) of broadcasting local
+      events to other nodes, defaults to 50 ms;
+    * `:max_remote_deltas` - the number of last broadcast deltas to keep for
+      catching up nodes that fell behind, defaults to 5.
 
   """
   defdelegate child_spec(opts), to: Firenest.ReplicatedState.Supervisor
diff --git a/test/firenest/synced_server_test.exs b/test/firenest/synced_server_test.exs
index a7fd032..fd3abeb 100644
--- a/test/firenest/synced_server_test.exs
+++ b/test/firenest/synced_server_test.exs
@@ -320,10 +320,10 @@ defmodule Firenest.SyncedServerTest do
   end
 
   defmodule Distributed do
-    use ExUnit.Case, async: true
+    use ExUnit.Case
 
     setup_all do
-      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 500)
+      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 5000)
       nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
       topology = Firenest.Test
       nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
diff --git a/test/shared/topology_test.exs b/test/shared/topology_test.exs
index 41a9540..9d92394 100644
--- a/test/shared/topology_test.exs
+++ b/test/shared/topology_test.exs
@@ -10,7 +10,7 @@ defmodule Firenest.TopologyTest do
   import Firenest.TestHelpers
 
   setup_all do
-    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
+    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 5000)
     topology = Firenest.Test
 
     {:ok,

From 853e808a883e4e106aed5cc383281b373b96550c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 21 Nov 2018 11:34:55 +0100
Subject: [PATCH 34/40] Refactor ReplicatedState.multicall

---
 lib/firenest/replicated_state.ex | 66 +++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 23 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 77b39f5..b1a6d26 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -34,8 +34,7 @@ defmodule Firenest.ReplicatedState do
 
   @type extra_action :: :delete | {:update_after, update :: term(), time :: pos_integer()}
 
-  @type server_opt ::
-    {:remote_changes, :ignore | :observe_collapsed | :observe_full}
+  @type server_opt :: {:remote_changes, :ignore | :observe_collapsed | :observe_full}
 
   @doc """
   Called when a partition starts up.
@@ -268,27 +267,48 @@ defmodule Firenest.ReplicatedState do
   # def dirty_list(server, group)
 
   defp multicall(servers, request, timeout) do
-    servers
-    |> Enum.map(fn server ->
-      pid = Process.whereis(server)
-      ref = Process.monitor(pid)
-      send(pid, {:"$gen_call", {self(), ref}, request})
-      ref
-    end)
-    |> Enum.map(fn ref ->
-      receive do
-        {^ref, reply} ->
-          Process.demonitor(ref, [:flush])
-          reply
-
-        {:DOWN, ^ref, _, _, reason} ->
-          exit(reason)
-      after
-        timeout ->
-          Process.demonitor(ref, [:flush])
-          exit(:timeout)
-      end
-    end)
+    timer_ref = :erlang.start_timer(timeout, self(), :timeout)
+
+    request_refs =
+      Enum.map(servers, fn server ->
+        pid = Process.whereis(server)
+        ref = Process.monitor(pid)
+        send(pid, {:"$gen_call", {self(), ref}, request})
+        ref
+      end)
+
+    collect_replies(request_refs, timer_ref)
+  end
+
+  defp collect_replies([], timer_ref) do
+    cancel_flush_timer(timer_ref)
+    []
+  end
+
+  defp collect_replies([ref | request_refs], timer_ref) do
+    receive do
+      {^ref, reply} ->
+        Process.demonitor(ref, [:flush])
+        [reply | collect_replies(request_refs, timer_ref)]
+
+      {:DOWN, ^ref, _, _, reason} ->
+        cancel_flush_timer(timer_ref)
+        Enum.each(request_refs, &Process.demonitor(&1, [:flush]))
+        exit(reason)
+
+      {:timeout, ^timer_ref, _} ->
+        Enum.each([ref | request_refs], &Process.demonitor(&1, [:flush]))
+        exit(:timeout)
+    end
+  end
+
+  defp cancel_flush_timer(timer_ref) do
+    :erlang.cancel_timer(timer_ref)
+    receive do
+      {:timeout, ^timer_ref, _} -> :ok
+    after
+      0 -> :ok
+    end
   end
 
   defp partition_info!(server, key) do

From 6741955e3df2570f9b6ae5df1c129db2d23f82a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Wed, 21 Nov 2018 12:06:26 +0100
Subject: [PATCH 35/40] Address code review feedback

---
 lib/firenest/replicated_state.ex        | 30 ++++++++++++-------------
 lib/firenest/replicated_state/remote.ex |  2 +-
 lib/firenest/replicated_state/server.ex | 20 ++++++++++-------
 lib/firenest/replicated_state/store.ex  | 21 +++++++++++++++--
 test/firenest/pub_sub_test.exs          |  2 +-
 5 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index b1a6d26..a6428ca 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -111,7 +111,7 @@ defmodule Firenest.ReplicatedState do
   It takes in the local delta value constructed in the `c:local_update/4`
   calls and returns a remote delta value that will be replicated to other
   servers. The value of the local delta is reset to the initial delta value
-  returned from the `c:local_join/2` callback.
+  returned from the `c:init/1` callback.
 
   In case the callback is not provided it defaults to just returning local delta.
   """
@@ -191,6 +191,8 @@ defmodule Firenest.ReplicatedState do
       catching up nodes that fell behind, defaults to 5.
 
   """
+  defdelegate start_link(opts), to: Firenest.ReplicatedState.Supervisor
+
   defdelegate child_spec(opts), to: Firenest.ReplicatedState.Supervisor
 
   @doc """
@@ -304,6 +306,7 @@ defmodule Firenest.ReplicatedState do
 
   defp cancel_flush_timer(timer_ref) do
     :erlang.cancel_timer(timer_ref)
+
     receive do
       {:timeout, ^timer_ref, _} -> :ok
     after
@@ -334,29 +337,26 @@ defmodule Firenest.ReplicatedState.Supervisor do
   @moduledoc false
   use Supervisor
 
-  def child_spec(opts) do
-    partitions = Keyword.get(opts, :partitions, 1)
+  def start_link(opts) do
     name = Keyword.fetch!(opts, :name)
-    topology = Keyword.fetch!(opts, :topology)
-    handler = Keyword.fetch!(opts, :handler)
     supervisor = Module.concat(name, "Supervisor")
-    arg = {partitions, name, topology, handler, opts}
-
-    %{
-      id: __MODULE__,
-      start: {Supervisor, :start_link, [__MODULE__, arg, [name: supervisor]]},
-      type: :supervisor
-    }
+    Supervisor.start_link(__MODULE__, {name, opts}, name: supervisor)
   end
 
-  def init({partitions, name, topology, handler, opts}) do
+  def init({name, opts}) do
+    partitions = Keyword.get(opts, :partitions, 1)
+    topology = Keyword.fetch!(opts, :topology)
+    handler = Keyword.fetch!(opts, :handler)
+
     names =
       for partition <- 0..(partitions - 1),
           do: Module.concat(name, "Partition" <> Integer.to_string(partition))
 
     children =
-      for name <- names,
-          do: {Firenest.ReplicatedState.Server, {name, topology, handler, opts}}
+      for name <- names do
+        spec = {Firenest.ReplicatedState.Server, {name, topology, handler, opts}}
+        Supervisor.child_spec(spec, id: name)
+      end
 
     :ets.new(name, [:named_table, :set, read_concurrency: true])
     :ets.insert(name, {:partitions, partitions, List.to_tuple(names)})
diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index af0451b..ff97783 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -45,7 +45,7 @@ defmodule Firenest.ReplicatedState.Remote do
   end
 
   def permdown(%__MODULE__{clocks: clocks} = state, ref) do
-    true = Map.has_key?(clocks, ref)
+    %{^ref => _} = clocks
     clocks = Map.delete(clocks, ref)
     {:delete, ref, %{state | clocks: clocks}}
   end
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index faf7a1c..407343f 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -5,13 +5,10 @@ defmodule Firenest.ReplicatedState.Server do
   alias Firenest.SyncedServer
   alias Firenest.ReplicatedState.{Store, Remote, Handler}
 
-  def child_spec({name, topology, handler, opts}) do
+  def start_link({name, topology, handler, opts}) do
     server_opts = [name: name, topology: topology]
 
-    %{
-      id: name,
-      start: {SyncedServer, :start_link, [__MODULE__, {name, handler, opts}, server_opts]}
-    }
+    SyncedServer.start_link(__MODULE__, {name, handler, opts}, server_opts)
   end
 
   @impl true
@@ -46,8 +43,11 @@ defmodule Firenest.ReplicatedState.Server do
     %{store: store, handler: handler, remote: remote} = state
 
     link(pid)
+    pid = resolve_pid(pid)
 
-    unless Store.present?(store, key, pid) do
+    if Store.present?(store, key, pid) do
+      {:reply, {:error, :already_present}, state}
+    else
       case Handler.local_put(handler, arg, key, pid) do
         {:put, value, delta, handler} ->
           remote = Remote.local_put(remote, key, pid, value)
@@ -60,13 +60,12 @@ defmodule Firenest.ReplicatedState.Server do
           handler = Handler.local_delete(handler, [value])
           {:reply, :ok, %{state | handler: handler, remote: remote}}
       end
-    else
-      {:reply, {:error, :already_present}, state}
     end
   end
 
   def handle_call({:update, key, pid, arg}, _from, state) do
     %{store: store, handler: handler, remote: remote} = state
+    pid = resolve_pid(pid)
 
     case Store.fetch(store, key, pid) do
       {:ok, value, delta} ->
@@ -101,6 +100,7 @@ defmodule Firenest.ReplicatedState.Server do
 
   def handle_call({:delete, key, pid}, _from, state) do
     %{store: store, handler: handler, remote: remote} = state
+    pid = resolve_pid(pid)
 
     case Store.local_delete(store, key, pid) do
       {:ok, value, store} ->
@@ -121,6 +121,7 @@ defmodule Firenest.ReplicatedState.Server do
 
   def handle_call({:delete, pid}, _from, state) do
     %{store: store, handler: handler} = state
+    pid = resolve_pid(pid)
 
     case Store.local_delete(store, pid) do
       {:ok, deletes, store} ->
@@ -274,6 +275,9 @@ defmodule Firenest.ReplicatedState.Server do
   defp link(:partition), do: true
   defp link(pid), do: Process.link(pid)
 
+  defp resolve_pid(:partition), do: self()
+  defp resolve_pid(pid), do: pid
+
   defp unlink_flush(:partition), do: true
 
   defp unlink_flush(pid) do
diff --git a/lib/firenest/replicated_state/store.ex b/lib/firenest/replicated_state/store.ex
index 132ede8..ed417dd 100644
--- a/lib/firenest/replicated_state/store.ex
+++ b/lib/firenest/replicated_state/store.ex
@@ -3,8 +3,25 @@ defmodule Firenest.ReplicatedState.Store do
 
   defstruct [:values, :pids]
 
-  # Common data exchange format:
-  # {{key, pid}, value}
+  # Local data is stored in the values table in the following format:
+  #
+  #     {{key, pid}, value, delta}
+  #
+  # Remote data is stored as:
+  #
+  #     {{key, pid}, value}
+  #
+  # To recognise the node that remote data is coming from we can use
+  # the `node/1` function in the match spec. This saves on space of
+  # explicitly storing node and whole-node operations should be rare
+  # in practice. We can ignore the version part of node_ref, since
+  # we're guaranteed to get a nodedown from the old version of the node
+  # before it comes back up with a new version - we can ignore the
+  # version when storing the data.
+  #
+  # The pids table stores data as {key, pid} with key position of 2.
+  # This allows having the same data format in both tables and save
+  # on some data shuffling.
 
   def new(name) do
     values = :ets.new(name, [:named_table, :protected, :ordered_set, read_concurrency: true])
diff --git a/test/firenest/pub_sub_test.exs b/test/firenest/pub_sub_test.exs
index 9a5ebf6..6aab7ea 100644
--- a/test/firenest/pub_sub_test.exs
+++ b/test/firenest/pub_sub_test.exs
@@ -7,7 +7,7 @@ defmodule Firenest.PubSubTest do
   import Firenest.TestHelpers
 
   setup_all do
-    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 500)
+    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 5000)
     nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
     pubsub = Firenest.Test.PubSub
     topology = Firenest.Test

From 37f8df446de80c07b690fd2ccc63ae455d085865 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 23 Nov 2018 13:16:03 +0100
Subject: [PATCH 36/40] More tests for ReplicatedState.Remote

---
 lib/firenest/replicated_state.ex              |  6 --
 lib/firenest/replicated_state/remote.ex       | 16 ++--
 lib/firenest/replicated_state/server.ex       |  2 +-
 .../firenest/replicated_state/remote_test.exs | 79 +++++++++++++++++--
 test/firenest/replicated_state/store_test.exs |  2 +-
 5 files changed, 85 insertions(+), 20 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index a6428ca..c7b1994 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -320,16 +320,10 @@ defmodule Firenest.ReplicatedState do
     ms = [{{:partitions, :"$1", :"$2"}, [], [extract]}]
     [info] = :ets.select(server, ms)
     info
-  catch
-    :error, :badarg ->
-      raise ArgumentError, "unknown key: #{inspect(server)}"
   end
 
   defp partition_infos!(server) do
     Tuple.to_list(:ets.lookup_element(server, :partitions, 3))
-  catch
-    :error, :badarg ->
-      raise ArgumentError, "unknown group: #{inspect(server)}"
   end
 end
 
diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index ff97783..dd669fd 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -16,6 +16,8 @@ defmodule Firenest.ReplicatedState.Remote do
 
   def clock(%__MODULE__{clock: clock}), do: clock
 
+  def clock_for(%__MODULE__{clocks: clocks}, ref), do: Map.fetch!(clocks, ref)
+
   # Reconnections are dead until we have permdown
   def up(%__MODULE__{clocks: clocks} = state, ref, clock) do
     case clocks do
@@ -52,10 +54,10 @@ defmodule Firenest.ReplicatedState.Remote do
 
   def catch_up(%__MODULE__{clock: current} = state, old_clock, state_getter)
       when old_clock < current do
-    %{deltas: deltas, tag: tag} = state
+    %{deltas: deltas(store: deltas, lowest: lowest), tag: tag} = state
 
-    if Map.has_key?(deltas, old_clock) do
-      {:deltas, tag, current, Enum.flat_map(old_clock..current, &Map.fetch!(deltas, &1))}
+    if old_clock >= lowest do
+      {:deltas, tag, current, Enum.map((old_clock + 1)..current, &Map.fetch!(deltas, &1))}
     else
       {:state_transfer, tag, current, state_getter.()}
     end
@@ -71,22 +73,22 @@ defmodule Firenest.ReplicatedState.Remote do
     deltas = store_deltas(deltas, new_clock, new_deltas)
     new_state = %{state | pending: %{}, clock: new_clock, broadcast: broadcast, deltas: deltas}
 
-    {{tag, clock, new_deltas}, new_state}
+    {{tag, new_clock, new_deltas}, new_state}
   end
 
   def handle_catch_up(%__MODULE__{tag: tag} = state, ref, {:deltas, tag, clock, deltas}) do
     %{clocks: clocks} = state
 
-    state = %{state | clocks: %{clocks | ref => clock}}
+    state = %{state | clocks: Map.put(clocks, ref, clock)}
     {puts, updates, deletes} = handle_deltas(tag, deltas)
     {:diff, puts, updates, deletes, state}
   end
 
-  def handle_catch_up(%__MODULE__{tag: tag} = state, from, {:state_transfer, tag, clock, data}) do
+  def handle_catch_up(%__MODULE__{tag: tag} = state, ref, {:state_transfer, tag, clock, data}) do
     %{clocks: clocks} = state
 
     case tag do
-      :ignore -> {:insert, data, %{state | clocks: %{clocks | from => clock}}}
+      :ignore -> {:replace, data, %{state | clocks: Map.put(clocks, ref, clock)}}
     end
   end
 
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 407343f..1b55d7d 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -214,7 +214,7 @@ defmodule Firenest.ReplicatedState.Server do
     %{remote: remote, store: store, handler: handler} = state
 
     case Remote.handle_catch_up(remote, from, data) do
-      {:insert, data, remote} ->
+      {:replace, data, remote} ->
         store = Store.remote_update(store, from, data)
         {:noreply, %{state | remote: remote, store: store}}
 
diff --git a/test/firenest/replicated_state/remote_test.exs b/test/firenest/replicated_state/remote_test.exs
index a3dfcbd..c4fd898 100644
--- a/test/firenest/replicated_state/remote_test.exs
+++ b/test/firenest/replicated_state/remote_test.exs
@@ -5,10 +5,14 @@ defmodule Firenest.ReplicatedState.RemoteTest do
 
   @moduletag remote_changes: :ignore
 
+  defmacrop assert_received_times(times, pattern) do
+    receives = List.duplicate(quote(do: assert_received(unquote(pattern))), times)
+    receives ++ [quote(do: refute_received(unquote(pattern)))]
+  end
+
   setup %{remote_changes: changes} do
     parent = self()
-    broadcast = fn -> send(parent, :broadcast) end
-    remote = Remote.new(changes, broadcast, 1)
+    %Remote{} = remote = Remote.new(changes, fn -> send(parent, :broadcast) end, 1)
     [remote: remote]
   end
 
@@ -27,8 +31,10 @@ defmodule Firenest.ReplicatedState.RemoteTest do
         |> Remote.local_put(:a, self(), 1)
         |> Remote.local_put(:b, self(), 2)
 
+      assert_received_times(1, :broadcast)
+
       {data, _remote} = Remote.broadcast(remote, &identity/1)
-      assert {:ignore, 0, {puts, _updates = [], _deletes = []}} = data
+      assert {:ignore, 1, {puts, _updates = [], _deletes = []}} = data
       assert [{{:b, self()}, 2}, {{:a, self()}, 1}] == puts
     end
 
@@ -40,7 +46,7 @@ defmodule Firenest.ReplicatedState.RemoteTest do
         |> Remote.local_update(:b, self(), 4, 5)
 
       {data, _remote} = Remote.broadcast(remote, &identity/1)
-      assert {:ignore, 0, {puts, updates, _deletes = []}} = data
+      assert {:ignore, 1, {puts, updates, _deletes = []}} = data
       assert [{{:b, self()}, 4}] == puts
       assert [{{:a, self()}, 2}] == updates
     end
@@ -55,10 +61,73 @@ defmodule Firenest.ReplicatedState.RemoteTest do
         |> Remote.local_delete(:c, self())
 
       {data, _remote} = Remote.broadcast(remote, &identity/1)
-      assert {:ignore, 0, {_puts = [], _updates = [], deletes}} = data
+      assert {:ignore, 1, {_puts = [], _updates = [], deletes}} = data
       assert [{:c, self()}, {:a, self()}] == deletes
     end
+
+    test "(handle_)catch_up/3 with deltas", %{remote: remote} do
+      other = Remote.new(:ignore, fn -> nil end, 1)
+
+      {_, remote} =
+        remote
+        |> Remote.local_put(:a, self(), 1)
+        |> Remote.broadcast(&identity/1)
+
+      assert {:catch_up, request, other} = connect(remote, :remote, other)
+      reply = Remote.catch_up(remote, request, fn -> flunk("should not get here") end)
+
+      assert {:diff, puts, updates, deletes, other} =
+               Remote.handle_catch_up(other, :remote, reply)
+
+      assert [{{:a, self()}, 1}] == puts
+      assert [] == updates
+      assert [] == deletes
+
+      assert Remote.clock_for(other, :remote) == Remote.clock(remote)
+    end
+
+    test "(handle_)catch_up/3 with state transfer", %{} do
+      remote = Remote.new(:ignore, fn -> nil end, 0)
+      other = Remote.new(:ignore, fn -> nil end, 1)
+
+      {_, remote} =
+        remote
+        |> Remote.local_put(:a, self(), 1)
+        |> Remote.broadcast(&identity/1)
+
+      assert {:catch_up, request, other} = connect(remote, :remote, other)
+      data = [{{:a, self()}, 1}]
+      reply = Remote.catch_up(remote, request, fn -> data end)
+
+      assert {:replace, ^data, other} = Remote.handle_catch_up(other, :remote, reply)
+      assert Remote.clock_for(other, :remote) == Remote.clock(remote)
+    end
+
+    test "handle_broadcast/3", %{remote: remote} do
+      other = Remote.new(:ignore, fn -> nil end, 1)
+
+      assert {:ok, other} = connect(remote, :remote, other)
+
+      {broadcast, remote} =
+        remote
+        |> Remote.local_put(:a, self(), 1)
+        |> Remote.broadcast(&identity/1)
+
+      assert {:diff, puts, updates, deletes, other} =
+               Remote.handle_broadcast(other, :remote, broadcast)
+
+      assert [{{:a, self()}, 1}] == puts
+      assert [] == updates
+      assert [] == deletes
+
+      assert Remote.clock_for(other, :remote) == Remote.clock(remote)
+    end
   end
 
   defp identity(x), do: x
+
+  defp connect(remote, id, other) do
+    clock = Remote.clock(remote)
+    Remote.up(other, id, clock)
+  end
 end
diff --git a/test/firenest/replicated_state/store_test.exs b/test/firenest/replicated_state/store_test.exs
index e3c4cc6..8d6bdc6 100644
--- a/test/firenest/replicated_state/store_test.exs
+++ b/test/firenest/replicated_state/store_test.exs
@@ -4,7 +4,7 @@ defmodule Firenest.ReplicatedState.StoreTest do
   alias Firenest.ReplicatedState.Store
 
   setup %{test: test} do
-    store = Store.new(test)
+    %Store{} = store = Store.new(test)
     other = spawn_link(fn -> Process.sleep(:infinity) end)
     [store: store, other: other]
   end

From 802bcf9e72bf5814aceb7cd3039ac6229d89461b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 23 Nov 2018 16:54:46 +0100
Subject: [PATCH 37/40] Distributed tests for ReplicatedState

---
 lib/firenest/replicated_state.ex              |   1 -
 lib/firenest/replicated_state/remote.ex       |   2 +-
 lib/firenest/replicated_state/server.ex       |  28 +-
 .../replicated_state/distributed_test.exs     |  94 ++++++
 .../firenest/replicated_state/remote_test.exs |   2 +-
 test/firenest/replicated_state_test.exs       | 212 +++++++------
 .../synced_server/distributed_test.exs        | 284 ++++++++++++++++++
 test/firenest/synced_server_test.exs          | 280 -----------------
 8 files changed, 514 insertions(+), 389 deletions(-)
 create mode 100644 test/firenest/replicated_state/distributed_test.exs
 create mode 100644 test/firenest/synced_server/distributed_test.exs

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index c7b1994..8b325ba 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -341,7 +341,6 @@ defmodule Firenest.ReplicatedState.Supervisor do
     partitions = Keyword.get(opts, :partitions, 1)
     topology = Keyword.fetch!(opts, :topology)
     handler = Keyword.fetch!(opts, :handler)
-
     names =
       for partition <- 0..(partitions - 1),
           do: Module.concat(name, "Partition" <> Integer.to_string(partition))
diff --git a/lib/firenest/replicated_state/remote.ex b/lib/firenest/replicated_state/remote.ex
index dd669fd..264bb2f 100644
--- a/lib/firenest/replicated_state/remote.ex
+++ b/lib/firenest/replicated_state/remote.ex
@@ -49,7 +49,7 @@ defmodule Firenest.ReplicatedState.Remote do
   def permdown(%__MODULE__{clocks: clocks} = state, ref) do
     %{^ref => _} = clocks
     clocks = Map.delete(clocks, ref)
-    {:delete, ref, %{state | clocks: clocks}}
+    {:delete, %{state | clocks: clocks}}
   end
 
   def catch_up(%__MODULE__{clock: current} = state, old_clock, state_getter)
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 1b55d7d..56ac529 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -11,6 +11,8 @@ defmodule Firenest.ReplicatedState.Server do
     SyncedServer.start_link(__MODULE__, {name, handler, opts}, server_opts)
   end
 
+  defstruct [:store, :handler, :remote]
+
   @impl true
   def init({name, handler, opts}) do
     Process.flag(:trap_exit, true)
@@ -28,7 +30,7 @@ defmodule Firenest.ReplicatedState.Server do
     remote = Remote.new(remote_changes, broadcast_fun, max_deltas)
 
     {:ok,
-     %{
+     %__MODULE__{
        store: store,
        handler: handler,
        remote: remote
@@ -40,7 +42,7 @@ defmodule Firenest.ReplicatedState.Server do
 
   @impl true
   def handle_call({:put, key, pid, arg}, _from, state) do
-    %{store: store, handler: handler, remote: remote} = state
+    %__MODULE__{store: store, handler: handler, remote: remote} = state
 
     link(pid)
     pid = resolve_pid(pid)
@@ -64,7 +66,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:update, key, pid, arg}, _from, state) do
-    %{store: store, handler: handler, remote: remote} = state
+    %__MODULE__{store: store, handler: handler, remote: remote} = state
     pid = resolve_pid(pid)
 
     case Store.fetch(store, key, pid) do
@@ -99,7 +101,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:delete, key, pid}, _from, state) do
-    %{store: store, handler: handler, remote: remote} = state
+    %__MODULE__{store: store, handler: handler, remote: remote} = state
     pid = resolve_pid(pid)
 
     case Store.local_delete(store, key, pid) do
@@ -120,7 +122,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:delete, pid}, _from, state) do
-    %{store: store, handler: handler} = state
+    %__MODULE__{store: store, handler: handler} = state
     pid = resolve_pid(pid)
 
     case Store.local_delete(store, pid) do
@@ -136,7 +138,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_call({:list, key}, _from, state) do
-    %{store: store} = state
+    %__MODULE__{store: store} = state
 
     read = fn -> Store.list(store, key) end
 
@@ -145,7 +147,7 @@ defmodule Firenest.ReplicatedState.Server do
 
   @impl true
   def handle_info({:EXIT, pid, reason}, state) do
-    %{store: store, handler: handler} = state
+    %__MODULE__{store: store, handler: handler} = state
 
     case Store.local_delete(store, pid) do
       {:ok, leaves, store} ->
@@ -159,7 +161,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_info(:broadcast_timeout, state) do
-    %{remote: remote, handler: handler} = state
+    %__MODULE__{remote: remote, handler: handler} = state
 
     prepare_delta = Handler.prepare_remote_delta_fun(handler)
     {data, remote} = Remote.broadcast(remote, prepare_delta)
@@ -168,7 +170,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_info({:update, key, pid, arg}, state) do
-    %{store: store, handler: handler} = state
+    %__MODULE__{store: store, handler: handler} = state
 
     case Store.fetch(store, key, pid) do
       {:ok, value, delta} ->
@@ -202,7 +204,7 @@ defmodule Firenest.ReplicatedState.Server do
 
   @impl true
   def handle_remote({:catch_up_req, data}, from, state) do
-    %{remote: remote, store: store} = state
+    %__MODULE__{remote: remote, store: store} = state
 
     get_all_local = fn -> Store.list_local(store) end
     reply = Remote.catch_up(remote, data, get_all_local)
@@ -211,7 +213,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_remote({:catch_up_rep, data}, from, state) do
-    %{remote: remote, store: store, handler: handler} = state
+    %__MODULE__{remote: remote, store: store, handler: handler} = state
 
     case Remote.handle_catch_up(remote, from, data) do
       {:replace, data, remote} ->
@@ -229,7 +231,7 @@ defmodule Firenest.ReplicatedState.Server do
   end
 
   def handle_remote({:broadcast, data}, from, state) do
-    %{remote: remote, store: store, hander: handler} = state
+    %__MODULE__{remote: remote, store: store, handler: handler} = state
 
     case Remote.handle_broadcast(remote, from, data) do
       {:diff, puts, updates, deletes, remote} ->
@@ -248,7 +250,7 @@ defmodule Firenest.ReplicatedState.Server do
 
   @impl true
   def handle_replica(change, remote_ref, state) do
-    %{remote: remote, store: store} = state
+    %__MODULE__{remote: remote, store: store} = state
 
     case remote_replica(change, remote_ref, remote) do
       {:ok, remote} ->
diff --git a/test/firenest/replicated_state/distributed_test.exs b/test/firenest/replicated_state/distributed_test.exs
new file mode 100644
index 0000000..a2c03cd
--- /dev/null
+++ b/test/firenest/replicated_state/distributed_test.exs
@@ -0,0 +1,94 @@
+defmodule Firenest.ReplicatedState.DistributedTest do
+  use ExUnit.Case
+
+  alias Firenest.Topology, as: T
+  alias Firenest.ReplicatedState, as: R
+
+  import Firenest.TestHelpers
+
+  setup_all do
+    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 5000)
+    nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
+    topology = Firenest.Test
+    server = Firenest.Test.ReplicatedState
+
+    %{start: start} =
+      R.child_spec(name: server, topology: topology, handler: Firenest.Test.EvalState)
+
+    Firenest.Test.start_link(nodes, start)
+    nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
+
+    {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, server: server}
+  end
+
+  test "remote join is propagated", config do
+    %{server: server, test: test, nodes: [second | _]} = config
+
+    quote do
+      spawn(fn ->
+        :ok = R.put(unquote(server), unquote(test), self(), :baz)
+        Process.sleep(:infinity)
+      end)
+    end
+    |> eval_on_node(second, config)
+
+    wait_until(fn -> R.list(server, test) == [:baz] end)
+  end
+
+  test "propages changes when nodes were disconnected", config do
+    %{topology: topology, server: server, test: test, nodes: [second, third]} = config
+    Process.register(self(), test)
+
+    quote do
+      spawn(fn ->
+        Process.register(self(), unquote(test))
+        :ok = R.put(unquote(server), unquote(test), self(), :baz)
+        Process.sleep(:infinity)
+      end)
+    end
+    |> eval_on_node(second, config)
+
+    quote(do: R.list(unquote(server), unquote(test)) == [:baz])
+    |> await_on_node(third, config)
+
+    quote do
+      T.disconnect(unquote(topology), elem(unquote(third), 0))
+      pid = Process.whereis(unquote(test))
+      :ok = R.delete(unquote(server), unquote(test), pid)
+
+      spawn(fn ->
+        :ok = R.put(unquote(server), unquote(test), self(), :bar)
+        Process.sleep(:infinity)
+      end)
+    end
+    |> eval_on_node(second, config)
+
+    quote(do: R.list(unquote(server), unquote(test)) == [])
+    |> await_on_node(third, config)
+
+    quote(do: T.connect(unquote(topology), elem(unquote(third), 0)))
+    |> eval_on_node(second, config)
+
+    quote(do: R.list(unquote(server), unquote(test)) == [:bar])
+    |> await_on_node(third, config)
+  end
+
+  defp eval_on_node(quoted, node, config) do
+    %{topology: topology, evaluator: evaluator} = config
+
+    T.send(topology, node, evaluator, {:eval_quoted, quoted})
+  end
+
+  defp await_on_node(quoted, node, config) do
+    %{topology: topology} = config
+    {:registered_name, name} = Process.info(self(), :registered_name)
+
+    quote do
+      wait_until(fn -> unquote(quoted) end)
+      T.broadcast(unquote(topology), unquote(name), :continue)
+    end
+    |> eval_on_node(node, config)
+
+    assert_receive :continue
+  end
+end
diff --git a/test/firenest/replicated_state/remote_test.exs b/test/firenest/replicated_state/remote_test.exs
index c4fd898..9ee61af 100644
--- a/test/firenest/replicated_state/remote_test.exs
+++ b/test/firenest/replicated_state/remote_test.exs
@@ -19,7 +19,7 @@ defmodule Firenest.ReplicatedState.RemoteTest do
   test "up and down", %{remote: remote} do
     assert {:ok, remote} = Remote.up(remote, :node1, 0)
     assert {:catch_up, 0, remote} = Remote.up(remote, :node2, 5)
-    assert {:delete, :node1, _remote} = Remote.down(remote, :node1)
+    assert {:delete, _remote} = Remote.down(remote, :node1)
   end
 
   describe "remote_changes: :ignore" do
diff --git a/test/firenest/replicated_state_test.exs b/test/firenest/replicated_state_test.exs
index 69f5129..0715b9c 100644
--- a/test/firenest/replicated_state_test.exs
+++ b/test/firenest/replicated_state_test.exs
@@ -1,11 +1,8 @@
 defmodule Firenest.ReplicatedStateTest do
   use ExUnit.Case, async: true
 
-  alias Firenest.Topology, as: T
   alias Firenest.ReplicatedState, as: R
 
-  import Firenest.TestHelpers
-
   setup_all do
     {:ok, topology: Firenest.Test, evaluator: Firenest.Test.Evaluator}
   end
@@ -83,7 +80,28 @@ defmodule Firenest.ReplicatedStateTest do
       assert [] == R.list(server, :foo)
     end
 
-    test "updates on a timeout with :update_after return", %{server: server} do
+    test "immediately deletes with :delete return and other keys", %{server: server} do
+      parent = self()
+
+      fun = fn delta, config ->
+        send(parent, {:local_put, delta, config})
+
+        delete = fn config ->
+          send(parent, {:local_delete, config})
+        end
+
+        {delta + 1, delete, :delete}
+      end
+
+      assert R.put(server, :bar, self(), 1) == :ok
+      assert R.put(server, :foo, self(), fun) == :ok
+      assert_received {:local_put, 0, _}
+      assert_received {:local_delete, _}
+
+      assert [] == R.list(server, :foo)
+    end
+
+    test "with :update_after return", %{server: server} do
       parent = self()
 
       fun = fn delta, config ->
@@ -105,6 +123,66 @@ defmodule Firenest.ReplicatedStateTest do
       assert_receive {:local_update, 1, 1, _}
       assert [2] == R.list(server, :foo)
     end
+
+    test "with :update_after return that deletes immediately", %{server: server} do
+      parent = self()
+
+      fun = fn delta, config ->
+        send(parent, {:local_put, delta, config})
+
+        update = fn delta, state, config ->
+          send(parent, {:local_update, delta, state, config})
+
+          delete = fn config ->
+            send(parent, {:local_delete, config})
+          end
+
+          {delta + 1, delete, :delete}
+        end
+
+        {delta + 1, 1, {:update_after, update, 50}}
+      end
+
+      assert R.put(server, :foo, self(), fun) == :ok
+      assert_received {:local_put, 0, _}
+      refute_received {:local_update, _, _, _}
+      assert [1] == R.list(server, :foo)
+
+      assert_receive {:local_update, 1, 1, _}
+      assert_received {:local_delete, _}
+      assert [] == R.list(server, :foo)
+    end
+
+    test "with :update_after return that deletes immediately with other keys", %{server: server} do
+      parent = self()
+
+      fun = fn delta, config ->
+        send(parent, {:local_put, delta, config})
+
+        update = fn delta, state, config ->
+          send(parent, {:local_update, delta, state, config})
+
+          delete = fn config ->
+            send(parent, {:local_delete, config})
+          end
+
+          {delta + 1, delete, :delete}
+        end
+
+        {delta + 1, 1, {:update_after, update, 50}}
+      end
+
+      assert R.put(server, :bar, self(), 1) == :ok
+      assert R.put(server, :foo, self(), fun) == :ok
+      assert_received {:local_put, 0, _}
+      refute_received {:local_update, _, _, _}
+      assert [1] == R.list(server, :foo)
+
+      assert_receive {:local_update, 1, 1, _}
+      assert_received {:local_delete, _}
+      assert [] == R.list(server, :foo)
+      assert [_] = R.list(server, :bar)
+    end
   end
 
   describe "delete/2" do
@@ -160,6 +238,17 @@ defmodule Firenest.ReplicatedStateTest do
       assert [2] == R.list(server, :foo)
     end
 
+    test "leaves other keys for same process intact", %{server: server} do
+      R.put(server, :foo, self(), 1)
+      R.put(server, :bar, self(), 2)
+      assert [_] = R.list(server, :foo)
+      assert [_] = R.list(server, :bar)
+
+      assert R.delete(server, :foo, self()) == :ok
+      assert [] = R.list(server, :foo)
+      assert [_] = R.list(server, :bar)
+    end
+
     test "does not remove non members", %{server: server} do
       [{_, pid, _, _}] = Supervisor.which_children(Module.concat(server, "Supervisor"))
       Process.link(pid)
@@ -221,7 +310,32 @@ defmodule Firenest.ReplicatedStateTest do
       assert [] == R.list(server, :foo)
     end
 
-    test "updates on a timeout with :update_after return", %{server: server} do
+    test "immediately deletes with :delete return and other keys", %{server: server} do
+      parent = self()
+
+      R.put(server, :bar, self(), 1)
+      R.put(server, :foo, self(), 1)
+      assert [1] = R.list(server, :foo)
+
+      fun = fn delta, state, config ->
+        send(parent, {:local_update, delta, state, config})
+
+        delete = fn config ->
+          send(parent, {:local_delete, config})
+        end
+
+        {delta + 1, delete, :delete}
+      end
+
+      assert R.update(server, :foo, self(), fun) == :ok
+      assert_received {:local_update, 0, 1, _}
+      assert_received {:local_delete, _}
+
+      assert [] = R.list(server, :foo)
+      assert [_] = R.list(server, :bar)
+    end
+
+    test "with :update_after return", %{server: server} do
       parent = self()
 
       R.put(server, :foo, self(), 1)
@@ -247,92 +361,4 @@ defmodule Firenest.ReplicatedStateTest do
       assert [3] == R.list(server, :foo)
     end
   end
-
-  # defmodule Distributed do
-  #   # We modify test topology, it can't be async
-  #   use ExUnit.Case
-
-  #   setup_all do
-  #     wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end)
-  #     nodes = [:"first@127.0.0.1", :"second@127.0.0.1", :"third@127.0.0.1"]
-  #     topology = Firenest.Test
-  #     server = Firenest.Test.ReplicatedServer
-  #     %{start: start} = R.child_spec(name: server, topology: topology)
-  #     Firenest.Test.start_link(nodes, start)
-  #     nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
-
-  #     {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, server: server}
-  #   end
-
-  #   test "remote join is propagated", config do
-  #     %{server: server, test: test, nodes: [second | _]} = config
-
-  #     quote do
-  #       spawn(fn ->
-  #         :ok = R.join(unquote(server), unquote(test), self(), :baz)
-  #         :timer.sleep(:infinity)
-  #       end)
-  #     end
-  #     |> eval_on_node(second, config)
-
-  #     wait_until(fn -> R.members(server, test) == [:baz] end)
-  #   end
-
-  #   test "propages changes when nodes were disconnected", config do
-  #     %{topology: topology, server: server, test: test, nodes: [second, third]} = config
-  #     Process.register(self(), test)
-
-  #     quote do
-  #       spawn(fn ->
-  #         Process.register(self(), unquote(test))
-  #         :ok = R.join(unquote(server), unquote(test), self(), :baz)
-  #         Process.sleep(:infinity)
-  #       end)
-  #     end
-  #     |> eval_on_node(second, config)
-
-  #     quote(do: R.members(unquote(server), unquote(test)) == [:baz])
-  #     |> await_on_node(third, config)
-
-  #     quote do
-  #       T.disconnect(unquote(topology), elem(unquote(third), 0))
-  #       pid = Process.whereis(unquote(test))
-  #       :ok = R.leave(unquote(server), unquote(test), pid)
-
-  #       spawn(fn ->
-  #         :ok = R.join(unquote(server), unquote(test), self(), :bar)
-  #         Process.sleep(:infinity)
-  #       end)
-  #     end
-  #     |> eval_on_node(second, config)
-
-  #     quote(do: R.members(unquote(server), unquote(test)) == [])
-  #     |> await_on_node(third, config)
-
-  #     quote(do: T.connect(unquote(topology), elem(unquote(third), 0)))
-  #     |> eval_on_node(second, config)
-
-  #     quote(do: R.members(unquote(server), unquote(test)) == [:bar])
-  #     |> await_on_node(third, config)
-  #   end
-
-  #   defp eval_on_node(quoted, node, config) do
-  #     %{topology: topology, evaluator: evaluator} = config
-
-  #     T.send(topology, node, evaluator, {:eval_quoted, quoted})
-  #   end
-
-  #   defp await_on_node(quoted, node, config) do
-  #     %{topology: topology} = config
-  #     {:registered_name, name} = Process.info(self(), :registered_name)
-
-  #     quote do
-  #       wait_until(fn -> unquote(quoted) end)
-  #       T.broadcast(unquote(topology), unquote(name), :continue)
-  #     end
-  #     |> eval_on_node(node, config)
-
-  #     assert_receive :continue
-  #   end
-  # end
 end
diff --git a/test/firenest/synced_server/distributed_test.exs b/test/firenest/synced_server/distributed_test.exs
new file mode 100644
index 0000000..84e9152
--- /dev/null
+++ b/test/firenest/synced_server/distributed_test.exs
@@ -0,0 +1,284 @@
+defmodule Firenest.SyncedServer.DistributedTest do
+  use ExUnit.Case
+
+  alias Firenest.SyncedServer, as: S
+  alias Firenest.Topology, as: T
+  alias Firenest.Test.EvalServer
+
+  import Firenest.TestHelpers
+
+  setup_all do
+    wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 5000)
+    nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
+    topology = Firenest.Test
+    nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
+    node = T.node(topology)
+    {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, node: node}
+  end
+
+  setup %{test: test, topology: topology} do
+    {:ok, pid} = S.start_link(EvalServer, 1, name: test, topology: topology)
+    mfa = &{S, :start_link, [EvalServer, &1, [name: test, topology: topology]]}
+    {:ok, mfa: mfa, pid: pid}
+  end
+
+  describe "handle_replica/3" do
+    test "both ends receive message", %{pid: pid, node: node} = config do
+      parent = self()
+
+      fun = fn status, replica ->
+        send(parent, {:replica, status, replica, 1})
+        {:noreply, 1}
+      end
+
+      remote_fun =
+        quote do
+          {:ok,
+           fn status, replica ->
+             send(unquote(parent), {:replica, status, replica, 2})
+             {:noreply, 2}
+           end}
+        end
+
+      send(pid, {:state, fun})
+      second = start_another(config, remote_fun)
+
+      assert_receive {:replica, {:up, _}, ^second, 1}
+      assert_receive {:replica, {:up, _}, ^node, 2}
+    end
+
+    test "{:noreply, state}", %{pid: pid} = config do
+      parent = self()
+
+      fun = fn status, replica ->
+        send(parent, {:replica, status, replica, 1})
+        {:noreply, 1}
+      end
+
+      send(pid, {:state, fun})
+      second = start_another(config)
+
+      assert_receive {:replica, {:up, _}, ^second, 1}
+      assert S.call(pid, :state) == 1
+    end
+
+    test "{:noreply, state, timeout}", %{pid: pid} = config do
+      parent = self()
+
+      fun = fn status, replica ->
+        timeout = fn ->
+          send(parent, {:timeout, 2})
+          {:noreply, 2}
+        end
+
+        send(parent, {:replica, status, replica, 1})
+
+        {:noreply, timeout, 0}
+      end
+
+      send(pid, {:state, fun})
+      second = start_another(config)
+
+      assert_receive {:replica, {:up, _}, ^second, 1}
+      assert_receive {:timeout, 2}
+      assert S.call(pid, :state) == 2
+    end
+
+    test "{:noreply, state, :hibernate}", %{pid: pid} = config do
+      parent = self()
+
+      fun = fn status, replica ->
+        send(parent, {:replica, status, replica, 1})
+        {:noreply, 1, :hibernate}
+      end
+
+      send(pid, {:state, fun})
+      second = start_another(config)
+
+      assert_hibernate pid
+      assert_receive {:replica, {:up, _}, ^second, 1}
+      assert S.call(pid, :state) == 1
+    end
+
+    test "{:stop, reason, state}", %{pid: pid} = config do
+      parent = self()
+      Process.flag(:trap_exit, true)
+
+      fun = fn status, replica ->
+        terminate = fn m ->
+          send(parent, {:terminate, m})
+        end
+
+        send(parent, {:replica, status, replica, 1})
+
+        {:stop, {:shutdown, terminate}, 1}
+      end
+
+      send(pid, {:state, fun})
+      second = start_another(config)
+
+      assert_receive {:replica, {:up, _}, ^second, 1}
+      assert_receive {:terminate, 1}
+      assert_receive {:EXIT, ^pid, {:shutdown, _}}
+    end
+
+    test "down", %{test: test} = config do
+      parent = self()
+
+      fun = fn status, replica ->
+        handle_remote = fn status, replica ->
+          send(parent, {:replica, status, replica, 2})
+          {:noreply, 2}
+        end
+
+        send(parent, {:replica, status, replica, 1})
+        {:noreply, handle_remote}
+      end
+
+      send(test, {:state, fun})
+      second = start_another(config)
+
+      cmd =
+        quote do
+          pid = Process.whereis(unquote(test))
+          Process.exit(pid, :kill)
+        end
+
+      assert send_eval(config, second, cmd) == :ok
+      assert_receive {:replica, {:up, _}, ^second, 1}
+      assert_receive {:replica, :down, ^second, 2}
+      assert S.call(test, :state) == 2
+    end
+  end
+
+  describe "handle_remote/3" do
+    setup config do
+      {:ok, second: start_another(config)}
+    end
+
+    test "{:noreply, state}", config do
+      %{second: second, node: node, test: test, pid: pid} = config
+      parent = self()
+
+      cmd =
+        quote do
+          fun = fn :info, state ->
+            handle_remote = fn from, n ->
+              send(unquote(parent), {:remote, from, n})
+              {:noreply, n + 1}
+            end
+
+            S.remote_send(unquote(node), handle_remote)
+            {:noreply, state}
+          end
+
+          send(unquote(test), fun)
+        end
+
+      assert send_eval(config, second, cmd) == :ok
+      assert_receive {:remote, ^second, 2}
+      assert S.call(pid, :state) == 3
+    end
+
+    test "{:noreply, state, timeout}", config do
+      %{second: second, test: test, pid: pid} = config
+      parent = self()
+
+      cmd =
+        quote do
+          fun = fn :info, state ->
+            handle_remote = fn from, n ->
+              timeout = fn ->
+                send(unquote(parent), {:timeout, n + 1})
+                {:noreply, n + 1}
+              end
+
+              send(unquote(parent), {:remote, from, n})
+              {:noreply, timeout, 0}
+            end
+
+            S.remote_broadcast(handle_remote)
+            {:noreply, state}
+          end
+
+          send(unquote(test), fun)
+        end
+
+      assert send_eval(config, second, cmd) == :ok
+      assert_receive {:remote, ^second, 2}
+      assert_receive {:timeout, 3}
+      assert S.call(pid, :state) == 3
+    end
+
+    test "{:noreply, state, :hibernate}", config do
+      %{second: second, test: test, pid: pid} = config
+      parent = self()
+
+      cmd =
+        quote do
+          fun = fn :info, state ->
+            handle_remote = fn from, n ->
+              send(unquote(parent), {:remote, from, n})
+              {:noreply, n + 1, :hibernate}
+            end
+
+            S.remote_broadcast(handle_remote)
+            {:noreply, state}
+          end
+
+          send(unquote(test), fun)
+        end
+
+      assert send_eval(config, second, cmd) == :ok
+      assert_receive {:remote, ^second, 2}
+      assert_hibernate pid
+      assert S.call(pid, :state) == 3
+    end
+
+    test "{:stop, reason, state}", config do
+      %{second: second, node: node, test: test, pid: pid} = config
+      parent = self()
+      Process.flag(:trap_exit, true)
+
+      cmd =
+        quote do
+          fun = fn :info, state ->
+            handle_remote = fn from, n ->
+              terminate = fn m ->
+                send(unquote(parent), {:terminate, m})
+              end
+
+              send(unquote(parent), {:remote, from, n})
+              {:stop, {:shutdown, terminate}, n + 1}
+            end
+
+            S.remote_send(unquote(node), handle_remote)
+            {:noreply, state}
+          end
+
+          send(unquote(test), fun)
+        end
+
+      assert send_eval(config, second, cmd) == :ok
+      assert_receive {:remote, ^second, 2}
+      assert_receive {:terminate, 3}
+      assert_receive {:EXIT, ^pid, {:shutdown, _}}
+    end
+  end
+
+  defp start_another(config) do
+    start_another(config, quote(do: {:ok, fn _, _ -> {:noreply, 1} end}))
+  end
+
+  defp start_another(config, initial_state) do
+    %{mfa: mfa, nodes: [second | _]} = config
+
+    Firenest.Test.start_link([elem(second, 0)], mfa.({:eval, initial_state}))
+    second
+  end
+
+  defp send_eval(config, to, cmd) do
+    %{evaluator: evaluator, topology: topology} = config
+    T.send(topology, to, evaluator, {:eval_quoted, cmd})
+  end
+end
diff --git a/test/firenest/synced_server_test.exs b/test/firenest/synced_server_test.exs
index fd3abeb..90a94f8 100644
--- a/test/firenest/synced_server_test.exs
+++ b/test/firenest/synced_server_test.exs
@@ -4,7 +4,6 @@ defmodule Firenest.SyncedServerTest do
   import ExUnit.CaptureIO
 
   alias Firenest.SyncedServer, as: S
-  alias Firenest.Topology, as: T
   alias Firenest.Test.EvalServer
 
   import Firenest.TestHelpers
@@ -318,283 +317,4 @@ defmodule Firenest.SyncedServerTest do
              end) =~ ~r"error.*GenServer.*\(stop\) shutdown: :terminate.*State: 3"sm
     end
   end
-
-  defmodule Distributed do
-    use ExUnit.Case
-
-    setup_all do
-      wait_until(fn -> Process.whereis(:firenest_topology_setup) == nil end, 5000)
-      nodes = [:"first@127.0.0.1", :"second@127.0.0.1"]
-      topology = Firenest.Test
-      nodes = for {name, _} = ref <- T.nodes(topology), name in nodes, do: ref
-      node = T.node(topology)
-      {:ok, topology: topology, evaluator: Firenest.Test.Evaluator, nodes: nodes, node: node}
-    end
-
-    setup %{test: test, topology: topology} do
-      {:ok, pid} = S.start_link(EvalServer, 1, name: test, topology: topology)
-      mfa = &{S, :start_link, [EvalServer, &1, [name: test, topology: topology]]}
-      {:ok, mfa: mfa, pid: pid}
-    end
-
-    describe "handle_replica/3" do
-      test "both ends receive message", %{pid: pid, node: node} = config do
-        parent = self()
-
-        fun = fn status, replica ->
-          send(parent, {:replica, status, replica, 1})
-          {:noreply, 1}
-        end
-
-        remote_fun =
-          quote do
-            {:ok,
-             fn status, replica ->
-               send(unquote(parent), {:replica, status, replica, 2})
-               {:noreply, 2}
-             end}
-          end
-
-        send(pid, {:state, fun})
-        second = start_another(config, remote_fun)
-
-        assert_receive {:replica, {:up, _}, ^second, 1}
-        assert_receive {:replica, {:up, _}, ^node, 2}
-      end
-
-      test "{:noreply, state}", %{pid: pid} = config do
-        parent = self()
-
-        fun = fn status, replica ->
-          send(parent, {:replica, status, replica, 1})
-          {:noreply, 1}
-        end
-
-        send(pid, {:state, fun})
-        second = start_another(config)
-
-        assert_receive {:replica, {:up, _}, ^second, 1}
-        assert S.call(pid, :state) == 1
-      end
-
-      test "{:noreply, state, timeout}", %{pid: pid} = config do
-        parent = self()
-
-        fun = fn status, replica ->
-          timeout = fn ->
-            send(parent, {:timeout, 2})
-            {:noreply, 2}
-          end
-
-          send(parent, {:replica, status, replica, 1})
-
-          {:noreply, timeout, 0}
-        end
-
-        send(pid, {:state, fun})
-        second = start_another(config)
-
-        assert_receive {:replica, {:up, _}, ^second, 1}
-        assert_receive {:timeout, 2}
-        assert S.call(pid, :state) == 2
-      end
-
-      test "{:noreply, state, :hibernate}", %{pid: pid} = config do
-        parent = self()
-
-        fun = fn status, replica ->
-          send(parent, {:replica, status, replica, 1})
-          {:noreply, 1, :hibernate}
-        end
-
-        send(pid, {:state, fun})
-        second = start_another(config)
-
-        assert_hibernate pid
-        assert_receive {:replica, {:up, _}, ^second, 1}
-        assert S.call(pid, :state) == 1
-      end
-
-      test "{:stop, reason, state}", %{pid: pid} = config do
-        parent = self()
-        Process.flag(:trap_exit, true)
-
-        fun = fn status, replica ->
-          terminate = fn m ->
-            send(parent, {:terminate, m})
-          end
-
-          send(parent, {:replica, status, replica, 1})
-
-          {:stop, {:shutdown, terminate}, 1}
-        end
-
-        send(pid, {:state, fun})
-        second = start_another(config)
-
-        assert_receive {:replica, {:up, _}, ^second, 1}
-        assert_receive {:terminate, 1}
-        assert_receive {:EXIT, ^pid, {:shutdown, _}}
-      end
-
-      test "down", %{test: test} = config do
-        parent = self()
-
-        fun = fn status, replica ->
-          handle_remote = fn status, replica ->
-            send(parent, {:replica, status, replica, 2})
-            {:noreply, 2}
-          end
-
-          send(parent, {:replica, status, replica, 1})
-          {:noreply, handle_remote}
-        end
-
-        send(test, {:state, fun})
-        second = start_another(config)
-
-        cmd =
-          quote do
-            pid = Process.whereis(unquote(test))
-            Process.exit(pid, :kill)
-          end
-
-        assert send_eval(config, second, cmd) == :ok
-        assert_receive {:replica, {:up, _}, ^second, 1}
-        assert_receive {:replica, :down, ^second, 2}
-        assert S.call(test, :state) == 2
-      end
-    end
-
-    describe "handle_remote/3" do
-      setup config do
-        {:ok, second: start_another(config)}
-      end
-
-      test "{:noreply, state}", config do
-        %{second: second, node: node, test: test, pid: pid} = config
-        parent = self()
-
-        cmd =
-          quote do
-            fun = fn :info, state ->
-              handle_remote = fn from, n ->
-                send(unquote(parent), {:remote, from, n})
-                {:noreply, n + 1}
-              end
-
-              S.remote_send(unquote(node), handle_remote)
-              {:noreply, state}
-            end
-
-            send(unquote(test), fun)
-          end
-
-        assert send_eval(config, second, cmd) == :ok
-        assert_receive {:remote, ^second, 2}
-        assert S.call(pid, :state) == 3
-      end
-
-      test "{:noreply, state, timeout}", config do
-        %{second: second, test: test, pid: pid} = config
-        parent = self()
-
-        cmd =
-          quote do
-            fun = fn :info, state ->
-              handle_remote = fn from, n ->
-                timeout = fn ->
-                  send(unquote(parent), {:timeout, n + 1})
-                  {:noreply, n + 1}
-                end
-
-                send(unquote(parent), {:remote, from, n})
-                {:noreply, timeout, 0}
-              end
-
-              S.remote_broadcast(handle_remote)
-              {:noreply, state}
-            end
-
-            send(unquote(test), fun)
-          end
-
-        assert send_eval(config, second, cmd) == :ok
-        assert_receive {:remote, ^second, 2}
-        assert_receive {:timeout, 3}
-        assert S.call(pid, :state) == 3
-      end
-
-      test "{:noreply, state, :hibernate}", config do
-        %{second: second, test: test, pid: pid} = config
-        parent = self()
-
-        cmd =
-          quote do
-            fun = fn :info, state ->
-              handle_remote = fn from, n ->
-                send(unquote(parent), {:remote, from, n})
-                {:noreply, n + 1, :hibernate}
-              end
-
-              S.remote_broadcast(handle_remote)
-              {:noreply, state}
-            end
-
-            send(unquote(test), fun)
-          end
-
-        assert send_eval(config, second, cmd) == :ok
-        assert_receive {:remote, ^second, 2}
-        assert_hibernate pid
-        assert S.call(pid, :state) == 3
-      end
-
-      test "{:stop, reason, state}", config do
-        %{second: second, node: node, test: test, pid: pid} = config
-        parent = self()
-        Process.flag(:trap_exit, true)
-
-        cmd =
-          quote do
-            fun = fn :info, state ->
-              handle_remote = fn from, n ->
-                terminate = fn m ->
-                  send(unquote(parent), {:terminate, m})
-                end
-
-                send(unquote(parent), {:remote, from, n})
-                {:stop, {:shutdown, terminate}, n + 1}
-              end
-
-              S.remote_send(unquote(node), handle_remote)
-              {:noreply, state}
-            end
-
-            send(unquote(test), fun)
-          end
-
-        assert send_eval(config, second, cmd) == :ok
-        assert_receive {:remote, ^second, 2}
-        assert_receive {:terminate, 3}
-        assert_receive {:EXIT, ^pid, {:shutdown, _}}
-      end
-    end
-
-    defp start_another(config) do
-      start_another(config, quote(do: {:ok, fn _, _ -> {:noreply, 1} end}))
-    end
-
-    defp start_another(config, initial_state) do
-      %{mfa: mfa, nodes: [second | _]} = config
-
-      Firenest.Test.start_link([elem(second, 0)], mfa.({:eval, initial_state}))
-      second
-    end
-
-    defp send_eval(config, to, cmd) do
-      %{evaluator: evaluator, topology: topology} = config
-      T.send(topology, to, evaluator, {:eval_quoted, cmd})
-    end
-  end
 end

From 445b77131d2145a9dbfd320b55a38e90dc4c90d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 23 Nov 2018 16:56:28 +0100
Subject: [PATCH 38/40] Don't pass funs between processes

---
 lib/firenest/replicated_state.ex        | 3 ++-
 lib/firenest/replicated_state/server.ex | 4 +---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 8b325ba..25c2623 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -262,7 +262,8 @@ defmodule Firenest.ReplicatedState do
   @spec list(server(), key()) :: [state()]
   def list(server, key) do
     partition = partition_info!(server, key)
-    SyncedServer.call(partition, {:list, key}).()
+    {m, f, args} = SyncedServer.call(partition, {:list, key})
+    apply(m, f, args)
   end
 
   # TODO
diff --git a/lib/firenest/replicated_state/server.ex b/lib/firenest/replicated_state/server.ex
index 56ac529..4b52758 100644
--- a/lib/firenest/replicated_state/server.ex
+++ b/lib/firenest/replicated_state/server.ex
@@ -140,9 +140,7 @@ defmodule Firenest.ReplicatedState.Server do
   def handle_call({:list, key}, _from, state) do
     %__MODULE__{store: store} = state
 
-    read = fn -> Store.list(store, key) end
-
-    {:reply, read, state}
+    {:reply, {Store, :list, [store, key]}, state}
   end
 
   @impl true

From 68710f666ae7e05273725712497334b2d3fe89a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 23 Nov 2018 17:00:32 +0100
Subject: [PATCH 39/40] Format & minor refactor

---
 lib/firenest/pub_sub.ex          | 3 +--
 lib/firenest/replicated_state.ex | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/firenest/pub_sub.ex b/lib/firenest/pub_sub.ex
index 9b1fbc8..a4f4253 100644
--- a/lib/firenest/pub_sub.ex
+++ b/lib/firenest/pub_sub.ex
@@ -269,8 +269,7 @@ defmodule Firenest.PubSub.Supervisor do
   end
 
   def init({pubsub, topology, options}) do
-    partitions =
-      options[:partitions] || System.schedulers_online() |> Kernel./(4) |> Float.ceil() |> trunc()
+    partitions = options[:partitions] || ceil(System.schedulers_online() / 4)
 
     {module, function} = options[:dispatcher] || {Firenest.PubSub.Dispatcher, :dispatch}
 
diff --git a/lib/firenest/replicated_state.ex b/lib/firenest/replicated_state.ex
index 25c2623..f35e107 100644
--- a/lib/firenest/replicated_state.ex
+++ b/lib/firenest/replicated_state.ex
@@ -342,6 +342,7 @@ defmodule Firenest.ReplicatedState.Supervisor do
     partitions = Keyword.get(opts, :partitions, 1)
     topology = Keyword.fetch!(opts, :topology)
     handler = Keyword.fetch!(opts, :handler)
+
     names =
       for partition <- 0..(partitions - 1),
           do: Module.concat(name, "Partition" <> Integer.to_string(partition))

From 1c8ef25abd1760f815daa40f0b19e08f593dffdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 23 Nov 2018 17:03:23 +0100
Subject: [PATCH 40/40] Documentation typos

---
 lib/firenest/pub_sub.ex | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/firenest/pub_sub.ex b/lib/firenest/pub_sub.ex
index a4f4253..d5992e7 100644
--- a/lib/firenest/pub_sub.ex
+++ b/lib/firenest/pub_sub.ex
@@ -41,7 +41,7 @@ defmodule Firenest.PubSub do
   end
 
   @doc """
-  Returns a child specifiction for pubsub with the given `options`.
+  Returns a child specification for pubsub with the given `options`.
 
   The `:name` and `:topology` keys are required as part of `options`.
   `:name` refers to the name of the pubsub to be started and `:topology`
@@ -119,7 +119,7 @@ defmodule Firenest.PubSub do
   Broadcasts the given `message` on `topic` in `pubsub`.
 
   Returns `:ok` or `{:error, reason}` in case of failures in
-  the distributed brodcast.
+  the distributed broadcast.
   """
   @spec broadcast(t, topic | [topic], term) :: :ok | {:error, term}
   def broadcast(pubsub, topic, message) when is_atom(pubsub) do
@@ -135,7 +135,7 @@ defmodule Firenest.PubSub do
   Broadcasts the given `message` on `topic` in `pubsub`.
 
   Returns `:ok` or raises `Firenest.PubSub.BroadcastError` in case of
-  failures in the distributed brodcast.
+  failures in the distributed broadcast.
   """
   @spec broadcast!(t, topic | [topic], term) :: :ok | no_return
   def broadcast!(pubsub, topic, message) do
@@ -153,7 +153,7 @@ defmodule Firenest.PubSub do
   are not delivered to the broadcasting process.
 
   Returns `:ok` or `{:error, reason}` in case of failures in
-  the distributed brodcast.
+  the distributed broadcast.
   """
   @spec broadcast_from(t, pid, topic | [topic], term) :: :ok | {:error, term()}
   def broadcast_from(pubsub, pid, topic, message) when is_atom(pubsub) and is_pid(pid) do
@@ -173,7 +173,7 @@ defmodule Firenest.PubSub do
   are not delivered to the broadcasting process.
 
   Returns `:ok` or raises `Firenest.PubSub.BroadcastError` in case of
-  failures in the distributed brodcast.
+  failures in the distributed broadcast.
   """
   @spec broadcast_from!(t, pid, topic | [topic], term) :: :ok | no_return
   def broadcast_from!(pubsub, pid, topic, message) do