From cbc414ce737dad730d343df1203b4e76f0440270 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Mon, 29 Jul 2013 09:55:23 -0700 Subject: [PATCH 01/15] Switch to macros for record definitions. --- include/riak_control.hrl | 15 ++++++----- src/riak_control_session.erl | 16 ++++++------ src/riak_control_wm_cluster.erl | 46 ++++++++++++++++----------------- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/include/riak_control.hrl b/include/riak_control.hrl index 52f24c8..267b650 100644 --- a/include/riak_control.hrl +++ b/include/riak_control.hrl @@ -58,8 +58,11 @@ partition :: integer(), owner :: owner(), vnodes :: services(), - handoffs :: handoffs() - }). + handoffs :: handoffs() }). + +-define(PARTITION_INFO, #partition_info). +-type partition() :: ?PARTITION_INFO{}. +-type partitions() :: [partition()]. -record(member_info, { node :: atom(), @@ -73,11 +76,11 @@ mem_used :: integer(), mem_erlang :: integer(), action :: action(), - replacement :: node() - }). + replacement :: node() }). --type partitions() :: [#partition_info{}]. --type members() :: [#member_info{}]. +-define(MEMBER_INFO, #member_info). +-type member() :: ?MEMBER_INFO{}. +-type members() :: [member()]. %% These two should always match, in terms of webmachine dispatcher %% logic, and ADMIN_BASE_PATH should always end with a / diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 9ac2ffd..05a15b6 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -272,7 +272,7 @@ update_partitions(State=#state{ring=Ring}) -> State#state{partitions=Partitions}. %% @doc Ping and retrieve vnode workers. --spec get_member_info({node(), status()}, ring()) -> #member_info{}. +-spec get_member_info({node(), status()}, ring()) -> member(). get_member_info(_Member={Node, Status}, Ring) -> RingSize = riak_core_ring:num_partitions(Ring), @@ -285,7 +285,7 @@ get_member_info(_Member={Node, Status}, Ring) -> %% try and get a list of all the vnodes running on the node case rpc:call(Node, riak_control_session, get_my_info, []) of {badrpc,nodedown} -> - #member_info{node = Node, + ?MEMBER_INFO{node = Node, status = Status, reachable = false, vnodes = [], @@ -293,29 +293,29 @@ get_member_info(_Member={Node, Status}, Ring) -> ring_pct = PctRing, pending_pct = PctPending}; {badrpc,_Reason} -> - #member_info{node = Node, + ?MEMBER_INFO{node = Node, status = incompatible, reachable = true, vnodes = [], handoffs = [], ring_pct = PctRing, pending_pct = PctPending}; - MemberInfo = #member_info{} -> + MemberInfo = ?MEMBER_INFO{} -> %% there is a race condition here, when a node is stopped %% gracefully (e.g. `riak stop`) the event will reach us %% before the node is actually down and the rpc call will %% succeed, but since it's shutting down it won't have any %% vnode workers running... - MemberInfo#member_info{status = Status, + MemberInfo?MEMBER_INFO{status = Status, ring_pct = PctRing, pending_pct = PctPending} end. %% @doc Return current nodes information. --spec get_my_info() -> #member_info{}. +-spec get_my_info() -> member(). get_my_info() -> {Total, Used} = get_my_memory(), - #member_info{node = node(), + ?MEMBER_INFO{node = node(), reachable = true, mem_total = Total, mem_used = Used, @@ -364,7 +364,7 @@ get_handoff_status() -> %% @doc Get handoffs for every node. -spec get_all_handoffs(#state{}) -> handoffs(). get_all_handoffs(#state{nodes=Members}) -> - lists:flatten([HS || #member_info{handoffs=HS} <- Members]). + lists:flatten([HS || ?MEMBER_INFO{handoffs=HS} <- Members]). %% @doc Get information for a particular index. -spec get_partition_details(#state{}, {integer(), term()}, handoffs()) diff --git a/src/riak_control_wm_cluster.erl b/src/riak_control_wm_cluster.erl index 9555f3d..64d5c35 100644 --- a/src/riak_control_wm_cluster.erl +++ b/src/riak_control_wm_cluster.erl @@ -196,7 +196,7 @@ to_json(ReqData, Context) -> %% Get the current node list. {ok, _V, Nodes} = riak_control_session:get_nodes(), - Current = [jsonify_node(Node) || Node=#member_info{} <- Nodes], + Current = [jsonify_node(Node) || Node=?MEMBER_INFO{} <- Nodes], %% Get the current list of planned changes and updated claim. Planned = case riak_control_session:get_plan() of @@ -215,7 +215,7 @@ to_json(ReqData, Context) -> {mochijson2:encode({struct,[{cluster,Clusters}]}), ReqData, Context}. %% @doc Generate a new "planned" cluster which outlines transitions. --spec merge_transitions(list(#member_info{}), list(), list()) -> +-spec merge_transitions(list(member()), list(), list()) -> [{struct, list()}]. merge_transitions(Nodes, Changes, Claim) -> lists:foldl(fun(Node, TransitionedNodes) -> @@ -224,32 +224,32 @@ merge_transitions(Nodes, Changes, Claim) -> end, [], Nodes). %% @doc Merge change into member info record. --spec apply_changes(#member_info{}, list(), list()) -> #member_info{}. +-spec apply_changes(member(), list(), list()) -> member(). apply_changes(Node, Changes, Claim) -> apply_status_change(apply_claim_change(Node, Claim), Changes). %% @doc Merge change into member info record. --spec apply_status_change(#member_info{}, list()) -> #member_info{}. +-spec apply_status_change(member(), list()) -> member(). apply_status_change(Node, Changes) -> - Name = Node#member_info.node, + Name = Node?MEMBER_INFO.node, case lists:keyfind(Name, 1, Changes) of false -> Node; {_, {Action, Replacement}} -> - Node#member_info{action=Action, replacement=Replacement}; + Node?MEMBER_INFO{action=Action, replacement=Replacement}; {_, Action} -> - Node#member_info{action=Action} + Node?MEMBER_INFO{action=Action} end. %% @doc Merge change into member info record. --spec apply_claim_change(#member_info{}, list()) -> #member_info{}. +-spec apply_claim_change(member(), list()) -> member(). apply_claim_change(Node, Claim) -> - Name = Node#member_info.node, + Name = Node?MEMBER_INFO.node, case lists:keyfind(Name, 1, Claim) of false -> - Node#member_info{ring_pct=0.0, pending_pct=0.0}; + Node?MEMBER_INFO{ring_pct=0.0, pending_pct=0.0}; {_, {_, Future}} -> %% @doc Hack until core returns normalized values. Normalized = if @@ -258,29 +258,29 @@ apply_claim_change(Node, Claim) -> true -> Future end, - Node#member_info{ring_pct=Normalized, pending_pct=Normalized} + Node?MEMBER_INFO{ring_pct=Normalized, pending_pct=Normalized} end. %% @doc Turn a node into a proper struct for serialization. --spec jsonify_node(#member_info{}) -> {struct, list()}. +-spec jsonify_node(member()) -> {struct, list()}. jsonify_node(Node) -> LWM=app_helper:get_env(riak_control,low_mem_watermark,0.1), - MemUsed = Node#member_info.mem_used, - MemTotal = Node#member_info.mem_total, - Reachable = Node#member_info.reachable, + MemUsed = Node?MEMBER_INFO.mem_used, + MemTotal = Node?MEMBER_INFO.mem_total, + Reachable = Node?MEMBER_INFO.reachable, LowMem = low_mem(Reachable, MemUsed, MemTotal, LWM), - {struct,[{"name",Node#member_info.node}, - {"status",Node#member_info.status}, + {struct,[{"name",Node?MEMBER_INFO.node}, + {"status",Node?MEMBER_INFO.status}, {"reachable",Reachable}, - {"ring_pct",Node#member_info.ring_pct}, - {"pending_pct",Node#member_info.pending_pct}, + {"ring_pct",Node?MEMBER_INFO.ring_pct}, + {"pending_pct",Node?MEMBER_INFO.pending_pct}, {"mem_total",MemTotal}, {"mem_used",MemUsed}, - {"mem_erlang",Node#member_info.mem_erlang}, + {"mem_erlang",Node?MEMBER_INFO.mem_erlang}, {"low_mem",LowMem}, - {"me",Node#member_info.node == node()}, - {"action",Node#member_info.action}, - {"replacement",Node#member_info.replacement}]}. + {"me",Node?MEMBER_INFO.node == node()}, + {"action",Node?MEMBER_INFO.action}, + {"replacement",Node?MEMBER_INFO.replacement}]}. %% @doc Given a struct/proplist that we've received via JSON, %% recursively turn the keys into atoms from binaries. From d9d7144841d7d23043d37358bc64d46788b39c6e Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Mon, 29 Jul 2013 09:57:20 -0700 Subject: [PATCH 02/15] Add default handling case for incompatible records. --- src/riak_control_session.erl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 05a15b6..1fb9141 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -308,7 +308,17 @@ get_member_info(_Member={Node, Status}, Ring) -> %% vnode workers running... MemberInfo?MEMBER_INFO{status = Status, ring_pct = PctRing, - pending_pct = PctPending} + pending_pct = PctPending}; + _ -> + %% default case where a record incompatibility causes a + %% failure matching the record format. + ?MEMBER_INFO{node = Node, + status = incompatible, + reachable = true, + vnodes = [], + handoffs = [], + ring_pct = PctRing, + pending_pct = PctPending} end. %% @doc Return current nodes information. From 8e34113ae70a2463c838bfacb579ce26fd7b2926 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Mon, 29 Jul 2013 10:22:42 -0700 Subject: [PATCH 03/15] Make the current 1.4.1 record be v2. --- include/riak_control.hrl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/include/riak_control.hrl b/include/riak_control.hrl index 267b650..62589e5 100644 --- a/include/riak_control.hrl +++ b/include/riak_control.hrl @@ -64,7 +64,21 @@ -type partition() :: ?PARTITION_INFO{}. -type partitions() :: [partition()]. +%% Riak 1.3 -record(member_info, + { node :: atom(), + status :: status(), + reachable :: boolean(), + vnodes :: vnodes(), + handoffs :: handoffs(), + ring_pct :: float(), + pending_pct :: float(), + mem_total :: integer(), + mem_used :: integer(), + mem_erlang :: integer() }). + +%% Riak 1.4.1+ +-record(member_info_v2, { node :: atom(), status :: status(), reachable :: boolean(), @@ -78,7 +92,7 @@ action :: action(), replacement :: node() }). --define(MEMBER_INFO, #member_info). +-define(MEMBER_INFO, #member_info_v2). -type member() :: ?MEMBER_INFO{}. -type members() :: [member()]. From 25ab9bee59c6a2e13a96326ccae18c1a401eaaae Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Mon, 29 Jul 2013 10:56:17 -0700 Subject: [PATCH 04/15] Update user interface to suppress RAM. Suppress the display of RAM and related information if the node is known to be incompatible. --- priv/admin/js/core.js | 19 +++++++++++++++++++ priv/admin/js/generated/templates.js | 4 ++-- .../js/templates/current_cluster_item.hbs | 2 +- .../admin/js/templates/current_nodes_item.hbs | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/priv/admin/js/core.js b/priv/admin/js/core.js index 27fd237..83e4f6d 100644 --- a/priv/admin/js/core.js +++ b/priv/admin/js/core.js @@ -59,6 +59,25 @@ minispade.register('core', function() { riak_pipe_vnode_status: DS.attr("string"), riak_search_vnode_status: DS.attr("string"), + /** + * Return status of whether the node is incompatible or not. + * + * @returns {Boolean} + */ + incompatible: function() { + return this.get('status') === 'incompatible'; + }.property('status'), + + /** + * Consider an available node one which is compatible, and + * reachable. + * + * @returns {Boolean} + */ + available: function() { + return !this.get('incompatible') && this.get('reachable'); + }.property('status', 'reachable'), + /** * Coerce vnode status into representations that are useful * for the user interface. diff --git a/priv/admin/js/generated/templates.js b/priv/admin/js/generated/templates.js index 37a383d..8e946ea 100644 --- a/priv/admin/js/generated/templates.js +++ b/priv/admin/js/generated/templates.js @@ -5,6 +5,6 @@ Ember.TEMPLATES['nodes'] = Ember.Handlebars.compile('

Current Ring

{{outlet partitionFilter}}
  • Prev
    • {{#each pages}} {{view RiakControl.PaginationItemView contentBinding="this"}} {{/each}}
  • Next
{{#collection RiakControl.PartitionView contentBinding="controller.paginatedContent"}} {{#with view.content}} {{/with}} {{#with view}} {{/with}} {{/collection}}

#

Owner Node

KV

Pipe

Search
{{i}}
{{node}}
{{index}}
 {{kvStatus}} {{pipeStatus}} {{searchStatus}}
  • Prev
    • {{#each pages}} {{view RiakControl.PaginationItemView contentBinding="this"}} {{/each}}
  • Next
'); Ember.TEMPLATES['partition_filter'] = Ember.Handlebars.compile('
Filter by...
{{view RiakControl.PartitionFilterSelectView id="filter" classNames="gui-dropdown" contentBinding="filters" optionLabelPath="content.name" optionValuePath="content.value" prompt="All" selectionBinding="controller.selectedPartitionFilter"}}
'); Ember.TEMPLATES['pagination_item'] = Ember.Handlebars.compile('{{#with view}} {{content.page_id}}{{/with}}'); -Ember.TEMPLATES['current_cluster_item'] = Ember.Handlebars.compile('{{#with view}}
{{#view RiakControl.CurrentClusterToggleView}}
{{/view}}
{{name}}
{{ringPctReadable}}%
{{#if reachable}}
{{memUsedReadable}}% {{else}}
{{/if}}

Use these actions to prepare this node to leave the cluster.

{{#if me}} Warning: This node is hosting Riak Control. If it leaves the cluster, Riak Control will be shut down. {{/if}}
{{#if controller.joiningNodesExist}}
Select Replacement Node
{{view RiakControl.ClusterItemSelectView prompt="Select Replacement Node" classNames="gui-dropdown" contentBinding="controller.joiningNodes" optionLabelPath="content.name"}}
{{else}}
No new nodes are currently staged to join.
{{/if}}
Click "STAGE" when you are ready to stage this action. STAGE
{{/with}}'); -Ember.TEMPLATES['current_nodes_item'] = Ember.Handlebars.compile('{{#with view}}
{{name}}
{{ringPctReadable}}%
{{#if reachable}}
{{memUsedReadable}}% {{else}}
{{/if}}
{{/with}}'); +Ember.TEMPLATES['current_cluster_item'] = Ember.Handlebars.compile('{{#with view}}
{{#view RiakControl.CurrentClusterToggleView}}
{{/view}}
{{name}}
{{ringPctReadable}}%
{{#if available}}
{{memUsedReadable}}% {{else}}
{{/if}}

Use these actions to prepare this node to leave the cluster.

{{#if me}} Warning: This node is hosting Riak Control. If it leaves the cluster, Riak Control will be shut down. {{/if}}
{{#if controller.joiningNodesExist}}
Select Replacement Node
{{view RiakControl.ClusterItemSelectView prompt="Select Replacement Node" classNames="gui-dropdown" contentBinding="controller.joiningNodes" optionLabelPath="content.name"}}
{{else}}
No new nodes are currently staged to join.
{{/if}}
Click "STAGE" when you are ready to stage this action. STAGE
{{/with}}'); +Ember.TEMPLATES['current_nodes_item'] = Ember.Handlebars.compile('{{#with view}}
{{name}}
{{ringPctReadable}}%
{{#if available}}
{{memUsedReadable}}% {{else}}
{{/if}}
{{/with}}'); Ember.TEMPLATES['staged_cluster_item'] = Ember.Handlebars.compile('{{#with view}}
{{name}}
{{ringPctReadable}}%
{{#if isAction}}
{{node_action}}
{{/if}} {{#if isReplaced}}
{{replacement}}
{{/if}}
{{/with}}'); diff --git a/priv/admin/js/templates/current_cluster_item.hbs b/priv/admin/js/templates/current_cluster_item.hbs index 07a2361..fb5795e 100644 --- a/priv/admin/js/templates/current_cluster_item.hbs +++ b/priv/admin/js/templates/current_cluster_item.hbs @@ -21,7 +21,7 @@
- {{#if reachable}} + {{#if available}}
diff --git a/priv/admin/js/templates/current_nodes_item.hbs b/priv/admin/js/templates/current_nodes_item.hbs index f5bec89..5080ebe 100644 --- a/priv/admin/js/templates/current_nodes_item.hbs +++ b/priv/admin/js/templates/current_nodes_item.hbs @@ -26,7 +26,7 @@
- {{#if reachable}} + {{#if available}}
From 9bf23ca54f5004b856590e5c902620eb32073b43 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Mon, 29 Jul 2013 10:56:37 -0700 Subject: [PATCH 05/15] Use macros, and handle race condition. --- src/riak_control_wm_nodes.erl | 49 +++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/riak_control_wm_nodes.erl b/src/riak_control_wm_nodes.erl index 4c7e266..33c2b6d 100644 --- a/src/riak_control_wm_nodes.erl +++ b/src/riak_control_wm_nodes.erl @@ -79,31 +79,46 @@ to_json(ReqData, Context) -> %% Get the current node list. {ok, _V, RawNodes} = riak_control_session:get_nodes(), - Nodes = [jsonify_node(Node) || Node=#member_info{} <- RawNodes], + Nodes = [jsonify_node(Node) || Node=?MEMBER_INFO{} <- RawNodes], Encoded = mochijson2:encode({struct, [{nodes, Nodes}]}), {Encoded, ReqData, Context}. %% @doc Turn a node into a proper struct for serialization. --spec jsonify_node(#member_info{}) -> {struct, list()}. +-spec jsonify_node(member()) -> {struct, list()}. jsonify_node(Node) -> LWM=app_helper:get_env(riak_control,low_mem_watermark,0.1), - MemUsed = Node#member_info.mem_used, - MemTotal = Node#member_info.mem_total, - Reachable = Node#member_info.reachable, - LowMem = case Reachable of - false -> - false; - true -> - 1.0 - (MemUsed/MemTotal) < LWM - end, - {struct,[{"name",Node#member_info.node}, - {"status",Node#member_info.status}, + MemUsed = Node?MEMBER_INFO.mem_used, + MemTotal = Node?MEMBER_INFO.mem_total, + Reachable = Node?MEMBER_INFO.reachable, + LowMem = low_mem(Reachable, MemUsed, MemTotal, LWM), + {struct,[{"name",Node?MEMBER_INFO.node}, + {"status",Node?MEMBER_INFO.status}, {"reachable",Reachable}, - {"ring_pct",Node#member_info.ring_pct}, - {"pending_pct",Node#member_info.pending_pct}, + {"ring_pct",Node?MEMBER_INFO.ring_pct}, + {"pending_pct",Node?MEMBER_INFO.pending_pct}, {"mem_total",MemTotal}, {"mem_used",MemUsed}, - {"mem_erlang",Node#member_info.mem_erlang}, + {"mem_erlang",Node?MEMBER_INFO.mem_erlang}, {"low_mem",LowMem}, - {"me",Node#member_info.node == node()}]}. + {"me",Node?MEMBER_INFO.node == node()}, + {"action",Node?MEMBER_INFO.action}, + {"replacement",Node?MEMBER_INFO.replacement}]}. + +%% @doc Determine if a node has low memory. +-spec low_mem(boolean(), number() | atom(), number() | atom(), number()) + -> boolean(). +low_mem(Reachable, MemUsed, MemTotal, LWM) -> + case Reachable of + false -> + false; + true -> + %% There is a race where the node is online, but memsup is + %% still starting so memory is unavailable. + case MemTotal of + undefined -> + false; + _ -> + 1.0 - (MemUsed/MemTotal) < LWM + end + end. From a5fc82613b683423b36cfc9c82445b2a64022a96 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 10:09:41 -0700 Subject: [PATCH 06/15] Use capabilities to enforce record format. --- src/riak_control_app.erl | 9 +++++- src/riak_control_session.erl | 56 ++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/riak_control_app.erl b/src/riak_control_app.erl index b313d7b..1df81e7 100644 --- a/src/riak_control_app.erl +++ b/src/riak_control_app.erl @@ -30,7 +30,14 @@ %% =================================================================== start(_StartType, _StartArgs) -> - riak_control_sup:start_link(). + case riak_control_sup:start_link() of + {error, Reason} -> + {error, Reason}; + {ok, _Pid} -> + riak_core_capability:register({riak_control, member_info_version}, + [v0, v1], + v0) + end. stop(_State) -> ok. diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 1fb9141..d3e472b 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -301,11 +301,12 @@ get_member_info(_Member={Node, Status}, Ring) -> ring_pct = PctRing, pending_pct = PctPending}; MemberInfo = ?MEMBER_INFO{} -> - %% there is a race condition here, when a node is stopped - %% gracefully (e.g. `riak stop`) the event will reach us - %% before the node is actually down and the rpc call will - %% succeed, but since it's shutting down it won't have any - %% vnode workers running... + MemberInfo?MEMBER_INFO{status = Status, + ring_pct = PctRing, + pending_pct = PctPending}; + MemberInfo0 = #member_info{} -> + %% Upgrade older member information record. + MemberInfo = upgrade_member_info(MemberInfo0), MemberInfo?MEMBER_INFO{status = Status, ring_pct = PctRing, pending_pct = PctPending}; @@ -325,13 +326,24 @@ get_member_info(_Member={Node, Status}, Ring) -> -spec get_my_info() -> member(). get_my_info() -> {Total, Used} = get_my_memory(), - ?MEMBER_INFO{node = node(), - reachable = true, - mem_total = Total, - mem_used = Used, - mem_erlang = proplists:get_value(total,erlang:memory()), - vnodes = riak_core_vnode_manager:all_vnodes(), - handoffs = get_handoff_status()}. + case riak_core_capability:get({riak_control, member_info_version}) of + v1 -> + ?MEMBER_INFO{node = node(), + reachable = true, + mem_total = Total, + mem_used = Used, + mem_erlang = proplists:get_value(total,erlang:memory()), + vnodes = riak_core_vnode_manager:all_vnodes(), + handoffs = get_handoff_status()}; + v0 -> + #member_info{node = node(), + reachable = true, + mem_total = Total, + mem_used = Used, + mem_erlang = proplists:get_value(total,erlang:memory()), + vnodes = riak_core_vnode_manager:all_vnodes(), + handoffs = get_handoff_status()} + end. %% @doc Return current nodes memory. -spec get_my_memory() -> {term(), term()}. @@ -479,3 +491,23 @@ maybe_stage_change(Node, Action, Replacement) -> stop -> rpc:call(Node, riak_core, stop, []) end. + +%% @doc Conditionally upgrade member info records once they cross node +%% boundaries. +-spec upgrade_member_info(member()) -> member(). +upgrade_member_info(MemberInfo = ?MEMBER_INFO{}) -> + MemberInfo; +upgrade_member_info(MemberInfo = #member_info{}) -> + ?MEMBER_INFO{ + node = MemberInfo?MEMBER_INFO.node, + status = MemberInfo?MEMBER_INFO.status, + reachable = MemberInfo?MEMBER_INFO.reachable, + vnodes = MemberInfo?MEMBER_INFO.vnodes, + handoffs = MemberInfo?MEMBER_INFO.handoffs, + ring_pct = MemberInfo?MEMBER_INFO.ring_pct, + pending_pct = MemberInfo?MEMBER_INFO.pending_pct, + mem_total = MemberInfo?MEMBER_INFO.mem_total, + mem_used = MemberInfo?MEMBER_INFO.mem_used, + mem_erlang = MemberInfo?MEMBER_INFO.mem_erlang, + action = MemberInfo?MEMBER_INFO.action, + replacement = MemberInfo?MEMBER_INFO.replacement}. From f3c2605ddcefbe04f9ae588ec9b893600fefb2ad Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 10:27:02 -0700 Subject: [PATCH 07/15] Prevent multiple resource calls. --- src/riak_control_session.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index d3e472b..44142a6 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -326,23 +326,26 @@ get_member_info(_Member={Node, Status}, Ring) -> -spec get_my_info() -> member(). get_my_info() -> {Total, Used} = get_my_memory(), + Handoffs = get_handoff_status(), + VNodes = riak_core_vnode_manager:all_vnodes(), + ErlangMemory = proplists:get_value(total,erlang:memory()), case riak_core_capability:get({riak_control, member_info_version}) of v1 -> ?MEMBER_INFO{node = node(), reachable = true, mem_total = Total, mem_used = Used, - mem_erlang = proplists:get_value(total,erlang:memory()), - vnodes = riak_core_vnode_manager:all_vnodes(), - handoffs = get_handoff_status()}; + mem_erlang = ErlangMemory, + vnodes = VNodes, + handoffs = Handoffs}; v0 -> #member_info{node = node(), reachable = true, mem_total = Total, mem_used = Used, - mem_erlang = proplists:get_value(total,erlang:memory()), - vnodes = riak_core_vnode_manager:all_vnodes(), - handoffs = get_handoff_status()} + mem_erlang = ErlangMemory, + vnodes = VNodes, + handoffs = Handoffs} end. %% @doc Return current nodes memory. From 0efa0afca4c2a5921f965b8431f23449e7dbad6b Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 11:03:10 -0700 Subject: [PATCH 08/15] Ensure start_link returns supervisor_ret. --- src/riak_control_app.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/riak_control_app.erl b/src/riak_control_app.erl index 1df81e7..be0a7c1 100644 --- a/src/riak_control_app.erl +++ b/src/riak_control_app.erl @@ -33,10 +33,11 @@ start(_StartType, _StartArgs) -> case riak_control_sup:start_link() of {error, Reason} -> {error, Reason}; - {ok, _Pid} -> + {ok, Pid} -> riak_core_capability:register({riak_control, member_info_version}, [v0, v1], - v0) + v0), + {ok, Pid} end. stop(_State) -> From 52f1508cb5d9bc90f9624ddc3247bd1c9ee76497 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 11:03:22 -0700 Subject: [PATCH 09/15] Pattern match against legacy record. --- src/riak_control_session.erl | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 44142a6..18ee6f4 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -502,15 +502,13 @@ upgrade_member_info(MemberInfo = ?MEMBER_INFO{}) -> MemberInfo; upgrade_member_info(MemberInfo = #member_info{}) -> ?MEMBER_INFO{ - node = MemberInfo?MEMBER_INFO.node, - status = MemberInfo?MEMBER_INFO.status, - reachable = MemberInfo?MEMBER_INFO.reachable, - vnodes = MemberInfo?MEMBER_INFO.vnodes, - handoffs = MemberInfo?MEMBER_INFO.handoffs, - ring_pct = MemberInfo?MEMBER_INFO.ring_pct, - pending_pct = MemberInfo?MEMBER_INFO.pending_pct, - mem_total = MemberInfo?MEMBER_INFO.mem_total, - mem_used = MemberInfo?MEMBER_INFO.mem_used, - mem_erlang = MemberInfo?MEMBER_INFO.mem_erlang, - action = MemberInfo?MEMBER_INFO.action, - replacement = MemberInfo?MEMBER_INFO.replacement}. + node = MemberInfo#member_info.node, + status = MemberInfo#member_info.status, + reachable = MemberInfo#member_info.reachable, + vnodes = MemberInfo#member_info.vnodes, + handoffs = MemberInfo#member_info.handoffs, + ring_pct = MemberInfo#member_info.ring_pct, + pending_pct = MemberInfo#member_info.pending_pct, + mem_total = MemberInfo#member_info.mem_total, + mem_used = MemberInfo#member_info.mem_used, + mem_erlang = MemberInfo#member_info.mem_erlang}. From 6941f07873a5737d67f3a51d7a553e8068ba0e29 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 11:13:27 -0700 Subject: [PATCH 10/15] Assume v0 record when the capability is accessed before registered. --- src/riak_control_session.erl | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 18ee6f4..0cfd49d 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -329,16 +329,29 @@ get_my_info() -> Handoffs = get_handoff_status(), VNodes = riak_core_vnode_manager:all_vnodes(), ErlangMemory = proplists:get_value(total,erlang:memory()), - case riak_core_capability:get({riak_control, member_info_version}) of - v1 -> - ?MEMBER_INFO{node = node(), - reachable = true, - mem_total = Total, - mem_used = Used, - mem_erlang = ErlangMemory, - vnodes = VNodes, - handoffs = Handoffs}; - v0 -> + try + case riak_core_capability:get({riak_control, member_info_version}) of + v1 -> + ?MEMBER_INFO{node = node(), + reachable = true, + mem_total = Total, + mem_used = Used, + mem_erlang = ErlangMemory, + vnodes = VNodes, + handoffs = Handoffs}; + v0 -> + #member_info{node = node(), + reachable = true, + mem_total = Total, + mem_used = Used, + mem_erlang = ErlangMemory, + vnodes = VNodes, + handoffs = Handoffs} + end + catch + _:{unknown_capability, {riak_control, member_info_version}} -> + %% Assume v0, when the capability hasn't been registered + %% when the RPC request arrives. #member_info{node = node(), reachable = true, mem_total = Total, From bf21e44218e361674d0117d2dacde92a0c552069 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 13:02:28 -0700 Subject: [PATCH 11/15] Handle mixed RPC calls. Handle race condition by triggering a badrpc exception, which has backwards compatibility with <= 1.3. Craft custom messages for 1.4.0 with the bad record format. --- src/riak_control_session.erl | 46 ++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 0cfd49d..40e685c 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -332,6 +332,7 @@ get_my_info() -> try case riak_core_capability:get({riak_control, member_info_version}) of v1 -> + %% >= 1.4.1, where we have the upgraded cluster record. ?MEMBER_INFO{node = node(), reachable = true, mem_total = Total, @@ -340,25 +341,13 @@ get_my_info() -> vnodes = VNodes, handoffs = Handoffs}; v0 -> - #member_info{node = node(), - reachable = true, - mem_total = Total, - mem_used = Used, - mem_erlang = ErlangMemory, - vnodes = VNodes, - handoffs = Handoffs} + %% pre-1.4.1. + handle_bad_record(Total, Used, ErlangMemory, VNodes, Handoffs) end catch - _:{unknown_capability, {riak_control, member_info_version}} -> - %% Assume v0, when the capability hasn't been registered - %% when the RPC request arrives. - #member_info{node = node(), - reachable = true, - mem_total = Total, - mem_used = Used, - mem_erlang = ErlangMemory, - vnodes = VNodes, - handoffs = Handoffs} + _:{unknown_capability, _} -> + %% capabilities are not registered yet. + erlang:throw({badrpc, unknown_capability}) end. %% @doc Return current nodes memory. @@ -510,7 +499,7 @@ maybe_stage_change(Node, Action, Replacement) -> %% @doc Conditionally upgrade member info records once they cross node %% boundaries. --spec upgrade_member_info(member()) -> member(). +-spec upgrade_member_info(member() | #member_info{}) -> member(). upgrade_member_info(MemberInfo = ?MEMBER_INFO{}) -> MemberInfo; upgrade_member_info(MemberInfo = #member_info{}) -> @@ -525,3 +514,24 @@ upgrade_member_info(MemberInfo = #member_info{}) -> mem_total = MemberInfo#member_info.mem_total, mem_used = MemberInfo#member_info.mem_used, mem_erlang = MemberInfo#member_info.mem_erlang}. + +%% @doc Handle incompatible record for the 1.4.0 release. +handle_bad_record(Total, Used, ErlangMemory, VNodes, Handoffs) -> + Counters = riak_core_capability:get({riak_kv, crdt}), + case lists:member(pncounter, Counters) of + true -> + %% 1.4.0, where we have a bad record. + {member_info, + node(), incompatible, true, VNodes, Handoffs, undefined, + undefined, Total, Used, ErlangMemory, undefined, + undefined}; + false -> + %% < 1.4.0, where we have the old style record. + #member_info{node = node(), + reachable = true, + mem_total = Total, + mem_used = Used, + mem_erlang = ErlangMemory, + vnodes = VNodes, + handoffs = Handoffs} + end. From ca589c6993970166449d02933a42456111995984 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 15:51:28 -0700 Subject: [PATCH 12/15] Use the member_info macro. --- src/riak_control_formatting.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_control_formatting.erl b/src/riak_control_formatting.erl index c4ee5eb..d6a33a5 100644 --- a/src/riak_control_formatting.erl +++ b/src/riak_control_formatting.erl @@ -39,7 +39,7 @@ action_result(_,Req,C) -> %% return a proplist of details for a given index node_ring_details (P=#partition_info{index=Index,vnodes=Vnodes},Nodes) -> case lists:keyfind(P#partition_info.owner,2,Nodes) of - #member_info{node=Node,status=Status,reachable=Reachable} -> + ?MEMBER_INFO{node=Node,status=Status,reachable=Reachable} -> Handoffs = P#partition_info.handoffs, VnodeStatuses = [{atom_to_list(VnodeName) ++ "_vnode_status", vnode_status(VnodeName, VnodeStatus, Handoffs)} From 18444f6ee3e293542a5189cfabf77092b9ea18bd Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 15:52:13 -0700 Subject: [PATCH 13/15] Don't attempt 1.4.0 detection. Capability negotiation has a race condition, so it can't be relied on to determine 1.4.0 as it will look like a 1.4.0 momentarily if the node is actual a 1.3.x node. --- src/riak_control_session.erl | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 40e685c..7d0b3f6 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -341,8 +341,7 @@ get_my_info() -> vnodes = VNodes, handoffs = Handoffs}; v0 -> - %% pre-1.4.1. - handle_bad_record(Total, Used, ErlangMemory, VNodes, Handoffs) + erlang:throw({badrpc, unknown_capability}) end catch _:{unknown_capability, _} -> @@ -514,24 +513,3 @@ upgrade_member_info(MemberInfo = #member_info{}) -> mem_total = MemberInfo#member_info.mem_total, mem_used = MemberInfo#member_info.mem_used, mem_erlang = MemberInfo#member_info.mem_erlang}. - -%% @doc Handle incompatible record for the 1.4.0 release. -handle_bad_record(Total, Used, ErlangMemory, VNodes, Handoffs) -> - Counters = riak_core_capability:get({riak_kv, crdt}), - case lists:member(pncounter, Counters) of - true -> - %% 1.4.0, where we have a bad record. - {member_info, - node(), incompatible, true, VNodes, Handoffs, undefined, - undefined, Total, Used, ErlangMemory, undefined, - undefined}; - false -> - %% < 1.4.0, where we have the old style record. - #member_info{node = node(), - reachable = true, - mem_total = Total, - mem_used = Used, - mem_erlang = ErlangMemory, - vnodes = VNodes, - handoffs = Handoffs} - end. From 8db02b740d329beea271e5b0fa44cdb3b22278b3 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 16:16:29 -0700 Subject: [PATCH 14/15] Fix ordering of negotiation. --- src/riak_control_app.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_control_app.erl b/src/riak_control_app.erl index be0a7c1..96ea017 100644 --- a/src/riak_control_app.erl +++ b/src/riak_control_app.erl @@ -35,7 +35,7 @@ start(_StartType, _StartArgs) -> {error, Reason}; {ok, Pid} -> riak_core_capability:register({riak_control, member_info_version}, - [v0, v1], + [v1, v0], v0), {ok, Pid} end. From e037a9fc225598a2f198a65e53dfe593b4000a48 Mon Sep 17 00:00:00 2001 From: Christopher Meiklejohn Date: Tue, 30 Jul 2013 16:50:43 -0700 Subject: [PATCH 15/15] Remove capability. Capability negotation is problematic because race conditions in negotation. --- src/riak_control_app.erl | 10 +--------- src/riak_control_session.erl | 36 +++++++++++++++--------------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/src/riak_control_app.erl b/src/riak_control_app.erl index 96ea017..b313d7b 100644 --- a/src/riak_control_app.erl +++ b/src/riak_control_app.erl @@ -30,15 +30,7 @@ %% =================================================================== start(_StartType, _StartArgs) -> - case riak_control_sup:start_link() of - {error, Reason} -> - {error, Reason}; - {ok, Pid} -> - riak_core_capability:register({riak_control, member_info_version}, - [v1, v0], - v0), - {ok, Pid} - end. + riak_control_sup:start_link(). stop(_State) -> ok. diff --git a/src/riak_control_session.erl b/src/riak_control_session.erl index 7d0b3f6..81acaf5 100644 --- a/src/riak_control_session.erl +++ b/src/riak_control_session.erl @@ -50,7 +50,8 @@ code_change/3]). %% exported for RPC calls. --export([get_my_info/0]). +-export([get_my_info/0, + get_my_info_v2/0]). -record(state, {vsn :: version(), services :: services(), @@ -283,7 +284,7 @@ get_member_info(_Member={Node, Status}, Ring) -> PctPending = length(FutureIndices) / RingSize, %% try and get a list of all the vnodes running on the node - case rpc:call(Node, riak_control_session, get_my_info, []) of + case rpc:call(Node, riak_control_session, get_my_info_v2, []) of {badrpc,nodedown} -> ?MEMBER_INFO{node = Node, status = Status, @@ -325,29 +326,22 @@ get_member_info(_Member={Node, Status}, Ring) -> %% @doc Return current nodes information. -spec get_my_info() -> member(). get_my_info() -> + erlang:throw({badrpc, incompatible}). + +%% @doc Return current nodes information. +-spec get_my_info_v2() -> member(). +get_my_info_v2() -> {Total, Used} = get_my_memory(), Handoffs = get_handoff_status(), VNodes = riak_core_vnode_manager:all_vnodes(), ErlangMemory = proplists:get_value(total,erlang:memory()), - try - case riak_core_capability:get({riak_control, member_info_version}) of - v1 -> - %% >= 1.4.1, where we have the upgraded cluster record. - ?MEMBER_INFO{node = node(), - reachable = true, - mem_total = Total, - mem_used = Used, - mem_erlang = ErlangMemory, - vnodes = VNodes, - handoffs = Handoffs}; - v0 -> - erlang:throw({badrpc, unknown_capability}) - end - catch - _:{unknown_capability, _} -> - %% capabilities are not registered yet. - erlang:throw({badrpc, unknown_capability}) - end. + ?MEMBER_INFO{node = node(), + reachable = true, + mem_total = Total, + mem_used = Used, + mem_erlang = ErlangMemory, + vnodes = VNodes, + handoffs = Handoffs}. %% @doc Return current nodes memory. -spec get_my_memory() -> {term(), term()}.