From f567007c519cbd1d91e69d1e59a0ae9a5c254cd2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 14 Feb 2013 12:26:23 -0600 Subject: [PATCH 01/10] Move bucket properties messages to riak.proto. --- src/riak.proto | 27 ++++++++++++++++++++++++++- src/riak_kv.proto | 25 ------------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/riak.proto b/src/riak.proto index 7edd173c..2a9898e6 100644 --- a/src/riak.proto +++ b/src/riak.proto @@ -22,7 +22,7 @@ */ /* -** Revision: 1.2 +** Revision: 1.4 */ // Java package specifiers @@ -47,3 +47,28 @@ message RpbPair { required bytes key = 1; optional bytes value = 2; } + + +// Get bucket properties request +message RpbGetBucketReq { + required bytes bucket = 1; +} + +// Get bucket properties response +message RpbGetBucketResp { + required RpbBucketProps props = 1; +} + +// Set bucket properties request +message RpbSetBucketReq { + required bytes bucket = 1; + required RpbBucketProps props = 2; +} + +// Set bucket properties response - no message defined, just send RpbSetBucketResp + +// Bucket properties +message RpbBucketProps { + optional uint32 n_val = 1; + optional bool allow_mult = 2; +} diff --git a/src/riak_kv.proto b/src/riak_kv.proto index b7622d79..2f421e95 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -122,25 +122,6 @@ message RpbListKeysResp { optional bool done = 2; } -// Get bucket properties request -message RpbGetBucketReq { - required bytes bucket = 1; -} - -// Get bucket properties response -message RpbGetBucketResp { - required RpbBucketProps props = 1; -} - -// Set bucket properties request -message RpbSetBucketReq { - required bytes bucket = 1; - required RpbBucketProps props = 2; -} - - -// Set bucket properties response - no message defined, just send RpbSetBucketResp - // Map/Reduce request message RpbMapRedReq { @@ -199,9 +180,3 @@ message RpbLink { optional bytes key = 2; optional bytes tag = 3; } - -// Bucket properties -message RpbBucketProps { - optional uint32 n_val = 1; - optional bool allow_mult = 2; -} From 1b170aafac8b3efba52278e46a356e982c2914eb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 14 Feb 2013 14:54:44 -0600 Subject: [PATCH 02/10] Add all known bucket properties to message format. --- src/riak.proto | 52 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/riak.proto b/src/riak.proto index 2a9898e6..747ef911 100644 --- a/src/riak.proto +++ b/src/riak.proto @@ -65,10 +65,60 @@ message RpbSetBucketReq { required RpbBucketProps props = 2; } -// Set bucket properties response - no message defined, just send RpbSetBucketResp +// Set bucket properties response - no message defined, just send +// RpbSetBucketResp + +// Module-Function pairs for commit hooks and other bucket properties +// that take functions +message RpbModFun { + required bytes module = 1; + required bytes function = 1; +} + +// A commit hook, which may either be a modfun or a JavaScript named +// function +message RpbCommitHook { + optional RpbModFun modfun = 1; + optional bytes name = 2; +} // Bucket properties message RpbBucketProps { + // Declared in riak_core_app optional uint32 n_val = 1; optional bool allow_mult = 2; + optional bool last_write_wins = 3; + repeated RpbCommitHook precommit = 4; + repeated RpbCommitHook postcommit = 5; + optional RpbModFun chash_keyfun = 6; + + // Declared in riak_kv_app + optional RpbModFun linkfun = 7; + optional uint32 old_vclock = 8; + optional uint32 young_vclock = 9; + optional uint32 big_vclock = 10; + optional uint32 small_vclock = 11; + optional uint32 pr = 12; + optional uint32 r = 13; + optional uint32 w = 14; + optional uint32 pw = 15; + optional uint32 dw = 16; + optional uint32 rw = 17; + optional bool basic_quorum = 18; + optional bool notfound_ok = 19; + + // Used by riak_kv_multibackend + optional bytes backend = 20; + + // Used by riak_search bucket fixup + optional bool search = 21; + + // Used by riak_repl bucket fixup + enum RpbReplMode { + OFF = 0; + REALTIME = 1; + FULLSYNC = 2; + BOTH = 3; + } + optional RpbReplMode repl = 22; } From 54a6c0e83c41ae99ba6e1b1f46e10a3ec5d56de2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 15 Feb 2013 14:01:15 -0600 Subject: [PATCH 03/10] Fix broken field number --- src/riak.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak.proto b/src/riak.proto index 747ef911..315da106 100644 --- a/src/riak.proto +++ b/src/riak.proto @@ -72,7 +72,7 @@ message RpbSetBucketReq { // that take functions message RpbModFun { required bytes module = 1; - required bytes function = 1; + required bytes function = 2; } // A commit hook, which may either be a modfun or a JavaScript named From 856674eed2e84b081d8b5ecd0959495b116ad74e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 15 Feb 2013 14:01:29 -0600 Subject: [PATCH 04/10] Fix typo in comment --- src/riak.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak.proto b/src/riak.proto index 315da106..3303ba07 100644 --- a/src/riak.proto +++ b/src/riak.proto @@ -107,7 +107,7 @@ message RpbBucketProps { optional bool basic_quorum = 18; optional bool notfound_ok = 19; - // Used by riak_kv_multibackend + // Used by riak_kv_multi_backend optional bytes backend = 20; // Used by riak_search bucket fixup From 02eb409a427e726b5dd6447eb47e7fd62ecf9be6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 15 Feb 2013 14:03:04 -0600 Subject: [PATCH 05/10] Move bucket property codec to riak_pb_codec and implement other properties --- src/riak_pb_codec.erl | 202 ++++++++++++++++++++++++++++++++++++++- src/riak_pb_kv_codec.erl | 30 ------ test/encoding_test.erl | 16 ++-- 3 files changed, 209 insertions(+), 39 deletions(-) diff --git a/src/riak_pb_codec.erl b/src/riak_pb_codec.erl index dd5d8bb1..f8e8a6c7 100644 --- a/src/riak_pb_codec.erl +++ b/src/riak_pb_codec.erl @@ -38,7 +38,33 @@ encode_bool/1, %% riakc_pb:pbify_bool decode_bool/1, %% riakc_pb:erlify_bool to_binary/1, %% riakc_pb:binary - to_list/1]). %% riakc_pb:any_to_list + to_list/1, %% riakc_pb:any_to_list + encode_bucket_props/1, %% riakc_pb:pbify_rpbbucketprops + decode_bucket_props/1, %% riakc_pb:erlify_rpbbucketprops + encode_modfun/1, + decode_modfun/2, + encode_commit_hooks/1, + decode_commit_hooks/1 + ]). + +%% @doc Bucket properties that store module/function pairs, e.g. +%% commit hooks, hash functions, link functions, will be in one of +%% these forms. More specifically: +%% +%% chash_keyfun :: {module(), function()} +%% linkfun :: {modfun, module(), function()} +%% precommit, postcommit :: [ {struct, [{binary(), binary()}]} ] +%% @end +-type modfun_property() :: {module(), function()} | {modfun, module(), function()} | {struct, [{binary(), binary()}]}. + +%% @doc Fields that can be specified in a commit hook must be +%% binaries. The valid values are <<"mod">>, <<"fun">>, <<"name">>. +%% Note that "mod" and "fun" must be used together, and "name" cannot +%% be used if the other two are present. +-type commit_hook_field() :: binary(). + +%% @doc Bucket properties that are commit hooks have this format. +-type commit_hook_property() :: [ {struct, [{commit_hook_field(), binary()}]} ]. %% @doc Create an iolist of msg code and protocol buffer %% message. Replaces `riakc_pb:encode/1'. @@ -191,3 +217,177 @@ encode_pair({K,V}) -> -spec decode_pair(#rpbpair{}) -> {string(), string()}. decode_pair(#rpbpair{key = K, value = V}) -> {K, V}. + + +%% @doc Convert an RpbBucketProps message to a property list +-spec decode_bucket_props(PBProps::#rpbbucketprops{} | undefined) -> [proplists:property()]. +decode_bucket_props(undefined) -> + []; +decode_bucket_props(#rpbbucketprops{n_val=N, + allow_mult=AM, + last_write_wins=LWW, + precommit=Pre, + postcommit=Post, + chash_keyfun=Chash, + linkfun=Link, + old_vclock=Old, + young_vclock=Young, + big_vclock=Big, + small_vclock=Small, + pr=PR, r=R, w=W, pw=PW, + dw=DW, rw=RW, + basic_quorum=BQ, + notfound_ok=NFOK, + backend=Backend, + search=Search, + repl=Repl + + }) -> + %% Extract numerical properties + [ {P,V} || {P,V} <- [ {n_val, N}, {old_vclock, Old}, {young_vclock, Young}, + {big_vclock, Big}, {small_vclock, Small} ], + V /= undefined ] ++ + %% Extract booleans + [ {BProp, decode_bool(Bool)} || + {BProp, Bool} <- [{allow_mult, AM}, {last_write_wins, LWW}, + {basic_quorum, BQ}, {notfound_ok, NFOK}, + {search, Search}], + Bool /= undefined ] ++ + + %% Extract commit hooks + [ {PrePostProp, decode_commit_hooks(CList)} || + {PrePostProp, CList} <- [{precommit, Pre}, {postcommit, Post}], + CList /= [] ] ++ + + %% Extract modfuns + [ {MFProp, decode_modfun(MF, MFProp)} || {MFProp, MF} <- [{chash_keyfun, Chash}, + {linkfun, Link}], + MF /= undefined ] ++ + + %% Extract backend + [ {backend, Backend} || is_binary(Backend) ] ++ + + %% Extract quora + [ {QProp, riak_pb_kv_codec:decode_quorum(Q)} || + {QProp, Q} <- [{pr, PR}, {r, R}, {w, W}, {pw, PW}, {dw, DW}, {rw, RW}], + Q /= undefined ] ++ + + %% Extract repl prop + [ {repl, Repl} || Repl /= undefined ]. + + +%% @doc Convert a property list to an RpbBucketProps message +-spec encode_bucket_props([proplists:property()]) -> PBProps::#rpbbucketprops{}. +encode_bucket_props(Props) -> + encode_bucket_props(Props, #rpbbucketprops{}). + +%% @doc Convert a property list to an RpbBucketProps message +%% @private +-spec encode_bucket_props([proplists:property()], PBPropsIn::#rpbbucketprops{}) -> PBPropsOut::#rpbbucketprops{}. +encode_bucket_props([], Pb) -> + Pb; +encode_bucket_props([{n_val, Nval} | Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{n_val = Nval}); +encode_bucket_props([{allow_mult, Flag} | Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{allow_mult = encode_bool(Flag)}); +encode_bucket_props([{last_write_wins, LWW}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{last_write_wins = encode_bool(LWW)}); +encode_bucket_props([{precommit, Precommit}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{precommit = encode_commit_hooks(Precommit)}); +encode_bucket_props([{postcommit, Postcommit}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{postcommit = encode_commit_hooks(Postcommit)}); +encode_bucket_props([{chash_keyfun, ModFun}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{chash_keyfun = encode_modfun(ModFun)}); +encode_bucket_props([{linkfun, ModFun}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{linkfun = encode_modfun(ModFun)}); +encode_bucket_props([{old_vclock, Num}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{old_vclock = Num}); +encode_bucket_props([{young_vclock, Num}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{young_vclock = Num}); +encode_bucket_props([{big_vclock, Num}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{big_vclock = Num}); +encode_bucket_props([{small_vclock, Num}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{small_vclock = Num}); +encode_bucket_props([{pr, Q}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{pr = riak_pb_kv_codec:encode_quorum(Q)}); +encode_bucket_props([{r, Q}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{r = riak_pb_kv_codec:encode_quorum(Q)}); +encode_bucket_props([{w, Q}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{w = riak_pb_kv_codec:encode_quorum(Q)}); +encode_bucket_props([{pw, Q}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{pw = riak_pb_kv_codec:encode_quorum(Q)}); +encode_bucket_props([{dw, Q}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{dw = riak_pb_kv_codec:encode_quorum(Q)}); +encode_bucket_props([{rw, Q}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{rw = riak_pb_kv_codec:encode_quorum(Q)}); +encode_bucket_props([{basic_quorum, BQ}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{basic_quorum = encode_bool(BQ)}); +encode_bucket_props([{notfound_ok, NFOK}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{notfound_ok = encode_bool(NFOK)}); +encode_bucket_props([{backend, B}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{backend = to_binary(B)}); +encode_bucket_props([{search, S}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{search = encode_bool(S)}); +encode_bucket_props([{repl, Atom}|Rest], Pb) -> + encode_bucket_props(Rest, Pb#rpbbucketprops{repl = Atom}); +encode_bucket_props([_Ignore|Rest], Pb) -> + %% Ignore any properties not explicitly part of the PB message + encode_bucket_props(Rest, Pb). + +%% @doc Converts a module-function specification into a RpbModFun message. +-spec encode_modfun(modfun_property()) -> #rpbmodfun{}. +encode_modfun({struct, Props}) -> + {<<"mod">>, Mod} = lists:keyfind(<<"mod">>, 1, Props), + {<<"fun">>, Fun} = lists:keyfind(<<"fun">>, 1, Props), + encode_modfun({Mod, Fun}); +encode_modfun({modfun, M, F}) -> + encode_modfun({M, F}); +encode_modfun({M, F}) -> + #rpbmodfun{module=to_binary(M), function=to_binary(F)}. + +%% @doc Converts an RpbModFun message into the appropriate format for +%% the given property. +-spec decode_modfun(#rpbmodfun{}, atom()) -> modfun_property(). +decode_modfun(MF, linkfun) -> + {M,F} = decode_modfun(MF, undefined), + {modfun, M, F}; +decode_modfun(#rpbmodfun{module=Mod, function=Fun}, commit_hook) -> + {struct, [{<<"mod">>, Mod}, {<<"fun">>, Fun}]}; +decode_modfun(#rpbmodfun{module=Mod, function=Fun}=MF, _Prop) -> + try + {binary_to_existing_atom(Mod, latin1), binary_to_existing_atom(Fun, latin1)} + catch + exit:{badarg,_} -> + error_logger:warning_msg("Creating new atoms from protobuffs message! ~p", [MF]), + {binary_to_atom(Mod, latin1), binary_to_atom(Fun, latin1)} + end. + +%% @doc Converts a list of commit hooks into a list of RpbCommitHook +%% messages. +-spec encode_commit_hooks([commit_hook_property()]) -> [ #rpbcommithook{} ]. +encode_commit_hooks(Hooks) -> + [ encode_commit_hook(Hook) || Hook <- Hooks ]. + +encode_commit_hook({struct, Props}=Hook) -> + FoundProps = [ lists:keymember(Field, 1, Props) || + Field <- [<<"mod">>, <<"fun">>, <<"name">>]], + case FoundProps of + [true, true, _] -> + #rpbcommithook{modfun=encode_modfun(Hook)}; + [false, false, true] -> + {<<"name">>, Name} = lists:keyfind(<<"name">>, 1, Props), + #rpbcommithook{name=to_binary(Name)}; + _ -> + erlang:error(badarg, [Hook]) + end. + +%% @doc Converts a list of RpbCommitHook messages into commit hooks. +-spec decode_commit_hooks([ #rpbcommithook{} ]) -> [ commit_hook_property() ]. +decode_commit_hooks(Hooks) -> + [ decode_commit_hook(Hook) || Hook <- Hooks, + Hook =/= #rpbcommithook{modfun=undefined, name=undefined} ]. + +decode_commit_hook(#rpbcommithook{modfun = Modfun}) when Modfun =/= undefined -> + decode_modfun(Modfun, commit_hook); +decode_commit_hook(#rpbcommithook{name = Name}) when Name =/= undefined -> + {struct, [{<<"name">>, Name}]}. diff --git a/src/riak_pb_kv_codec.erl b/src/riak_pb_kv_codec.erl index 824be1ae..93d38ab4 100644 --- a/src/riak_pb_kv_codec.erl +++ b/src/riak_pb_kv_codec.erl @@ -42,8 +42,6 @@ decode_pair/1, %% riakc_pb:erlify_rpbpair encode_link/1, %% riakc_pb:pbify_rpblink decode_link/1, %% riakc_pb:erlify_rpblink - encode_bucket_props/1, %% riakc_pb:pbify_rpbbucketprops - decode_bucket_props/1, %% riakc_pb:erlify_rpbbucketprops encode_quorum/1, decode_quorum/1 %% riak_kv_pb_socket:normalize_rw_value ]). @@ -206,34 +204,6 @@ encode_link({{B,K},T}) -> decode_link(#rpblink{bucket = B, key = K, tag = T}) -> {{B,K},T}. - -%% @doc Convert an RpbBucketProps message to a property list --spec decode_bucket_props(PBProps::#rpbbucketprops{} | undefined) -> [proplists:property()]. -decode_bucket_props(undefined) -> - []; -decode_bucket_props(#rpbbucketprops{n_val=N, allow_mult=AM}) -> - [ {n_val, N} || N /= undefined ] ++ - [ {allow_mult, decode_bool(AM)} || AM /= undefined ]. - -%% @doc Convert a property list to an RpbBucketProps message --spec encode_bucket_props([proplists:property()]) -> PBProps::#rpbbucketprops{}. -encode_bucket_props(Props) -> - encode_bucket_props(Props, #rpbbucketprops{}). - -%% @doc Convert a property list to an RpbBucketProps message -%% @private --spec encode_bucket_props([proplists:property()], PBPropsIn::#rpbbucketprops{}) -> PBPropsOut::#rpbbucketprops{}. -encode_bucket_props([], Pb) -> - Pb; -encode_bucket_props([{n_val, Nval} | Rest], Pb) -> - encode_bucket_props(Rest, Pb#rpbbucketprops{n_val = Nval}); -encode_bucket_props([{allow_mult, Flag} | Rest], Pb) -> - encode_bucket_props(Rest, Pb#rpbbucketprops{allow_mult = encode_bool(Flag)}); -encode_bucket_props([_Ignore|Rest], Pb) -> - %% Ignore any properties not explicitly part of the PB message - encode_bucket_props(Rest, Pb). - - %% @doc Encode a symbolic or numeric quorum value into a Protocol %% Buffers value -spec encode_quorum(symbolic_quorum() | non_neg_integer()) -> non_neg_integer(). diff --git a/test/encoding_test.erl b/test/encoding_test.erl index 57c5188a..05d9a8d9 100644 --- a/test/encoding_test.erl +++ b/test/encoding_test.erl @@ -91,10 +91,10 @@ pb_test_() -> ?_test(begin Props = [{n_val, 99}, {allow_mult, true}], - Props2 = riak_pb_kv_codec:decode_bucket_props( - riak_kv_pb:decode_rpbbucketprops( - iolist_to_binary(riak_kv_pb:encode_rpbbucketprops( - riak_pb_kv_codec:encode_bucket_props(Props))))), + Props2 = riak_pb_codec:decode_bucket_props( + riak_pb:decode_rpbbucketprops( + iolist_to_binary(riak_pb:encode_rpbbucketprops( + riak_pb_codec:encode_bucket_props(Props))))), MdSame = (lists:sort(Props) =:= lists:sort(Props2)), ?assertEqual(true, MdSame) @@ -103,10 +103,10 @@ pb_test_() -> ?_test(begin Props = [{n_val, 33}, {allow_mult, false}], - Props2 = riak_pb_kv_codec:decode_bucket_props( - riak_kv_pb:decode_rpbbucketprops( - iolist_to_binary(riak_kv_pb:encode_rpbbucketprops( - riak_pb_kv_codec:encode_bucket_props(Props))))), + Props2 = riak_pb_codec:decode_bucket_props( + riak_pb:decode_rpbbucketprops( + iolist_to_binary(riak_pb:encode_rpbbucketprops( + riak_pb_codec:encode_bucket_props(Props))))), MdSame = (lists:sort(Props) =:= lists:sort(Props2)), ?assertEqual(true, MdSame) From 029b02b2fd88efbc97fa5763f7f756cdc0ee72c9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 15 Feb 2013 15:12:01 -0600 Subject: [PATCH 06/10] Fix repl enum and add bucket properties codec quickcheck --- .gitignore | 3 + rebar.config | 2 + src/riak.proto | 8 +-- test/bucket_props_codec_eqc.erl | 112 ++++++++++++++++++++++++++++++++ test/encoding_test.erl | 24 ------- 5 files changed, 121 insertions(+), 28 deletions(-) create mode 100644 test/bucket_props_codec_eqc.erl diff --git a/.gitignore b/.gitignore index f7f2de4e..2389bd32 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ ebin/*.beam ebin/riak_pb.app include/*_pb.hrl doc/* +.qc +.eqc-info +current_counterexample.eqc # Python riak_pb.egg-info diff --git a/rebar.config b/rebar.config index d711772c..dd5f8d33 100644 --- a/rebar.config +++ b/rebar.config @@ -4,5 +4,7 @@ {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", "master"}} ]}. +{eunit_opts, [verbose]}. + %% Fixes attempted removal of riak_pb directory by rebar_escripter {escript_name, "doesnothavescript"}. diff --git a/src/riak.proto b/src/riak.proto index 3303ba07..cf1a773c 100644 --- a/src/riak.proto +++ b/src/riak.proto @@ -115,10 +115,10 @@ message RpbBucketProps { // Used by riak_repl bucket fixup enum RpbReplMode { - OFF = 0; - REALTIME = 1; - FULLSYNC = 2; - BOTH = 3; + off = 0; + realtime = 1; + fullsync = 2; + both = 3; } optional RpbReplMode repl = 22; } diff --git a/test/bucket_props_codec_eqc.erl b/test/bucket_props_codec_eqc.erl new file mode 100644 index 00000000..da505fe0 --- /dev/null +++ b/test/bucket_props_codec_eqc.erl @@ -0,0 +1,112 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(bucket_props_codec_eqc). + +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(QC_OUT(P), eqc:on_output(fun(F,TL) -> + io:format(user, F, TL) + end, P)). + +bucket_codec_test_() -> + [{"bucket properties encode decode", + ?_test(begin + quickcheck(?QC_OUT(numtests(1000, prop_codec()))) + end)}]. + +prop_codec() -> + ?FORALL(Props, sortuniq(list(bucket_prop())), + ?WHENFAIL(begin + io:format("Props: ~p~n~n", [Props]) + end, + begin + Props2 = riak_pb_codec:decode_bucket_props( + riak_pb:decode_rpbbucketprops( + iolist_to_binary(riak_pb:encode_rpbbucketprops( + riak_pb_codec:encode_bucket_props(Props))))), + Props =:= lists:sort(Props2) + end)). + +bucket_prop() -> + oneof([num(n_val), + flag(allow_mult), + flag(last_write_wins), + commit(precommit), + commit(postcommit), + chash(), + linkfun(), + num(old_vclock), + num(young_vclock), + num(big_vclock), + num(small_vclock), + quorum(pr), + quorum(r), + quorum(w), + quorum(pw), + quorum(dw), + quorum(rw), + flag(basic_quorum), + flag(notfound_ok), + backend(), + flag(search), + repl()]). + +sortuniq(Gen) -> + ?LET(L, Gen, lists:ukeysort(1,L)). + +flag(Prop) -> + ?LET(B, bool(), {Prop, B}). + +num(Prop) -> + ?LET(N, nat(), {Prop, N}). + +quorum(Prop) -> + ?LET(V, oneof([one, quorum, all, default, nat()]), {Prop, V}). + +backend() -> + ?LET(B, non_empty(binary()), {backend, B}). + +repl() -> + ?LET(R, oneof([off, realtime, fullsync, both]), {repl, R}). + +commit(Prop) -> + ?LET(C, non_empty(list(commit_hook())), {Prop, C}). + +commit_hook() -> + ?LET(H, oneof([modfun_hook(), name_hook()]), {struct, H}). + +modfun_hook() -> + ?LET({M,F}, {non_empty(binary()), non_empty(binary())}, + [{<<"mod">>, M}, {<<"fun">>, F}]). + +name_hook() -> + ?LET(N, non_empty(binary()), [{<<"name">>, N}]). + +chash() -> + ?LET({M,F}, {atom(), atom()}, {chash_keyfun,{M,F}}). + +atom() -> + ?LET(B, non_empty(binary()), binary_to_atom(B, latin1)). + +linkfun() -> + ?LET({M,F}, {atom(), atom()}, {linkfun, {modfun, M, F}}). diff --git a/test/encoding_test.erl b/test/encoding_test.erl index 05d9a8d9..4004839b 100644 --- a/test/encoding_test.erl +++ b/test/encoding_test.erl @@ -86,30 +86,6 @@ pb_test_() -> {"msg code encode decode", ?_test(begin msg_code_encode_decode(0) - end)}, - {"bucket props encode decode", - ?_test(begin - Props = [{n_val, 99}, - {allow_mult, true}], - Props2 = riak_pb_codec:decode_bucket_props( - riak_pb:decode_rpbbucketprops( - iolist_to_binary(riak_pb:encode_rpbbucketprops( - riak_pb_codec:encode_bucket_props(Props))))), - MdSame = (lists:sort(Props) =:= - lists:sort(Props2)), - ?assertEqual(true, MdSame) - end)}, - {"bucket props encode decode 2", - ?_test(begin - Props = [{n_val, 33}, - {allow_mult, false}], - Props2 = riak_pb_codec:decode_bucket_props( - riak_pb:decode_rpbbucketprops( - iolist_to_binary(riak_pb:encode_rpbbucketprops( - riak_pb_codec:encode_bucket_props(Props))))), - MdSame = (lists:sort(Props) =:= - lists:sort(Props2)), - ?assertEqual(true, MdSame) end)} ]. From 5f0e2ae3f8dc6d0e60543be83135ff63d535dce4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 15 Feb 2013 15:30:06 -0600 Subject: [PATCH 07/10] Bump versions in .protos and ifdef the quickcheck --- src/riak_kv.proto | 2 +- src/riak_search.proto | 2 +- test/bucket_props_codec_eqc.erl | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 2f421e95..b6fd8695 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -22,7 +22,7 @@ */ /* -** Revision: 1.2 +** Revision: 1.4 */ // Java package specifiers diff --git a/src/riak_search.proto b/src/riak_search.proto index d196800b..97541feb 100644 --- a/src/riak_search.proto +++ b/src/riak_search.proto @@ -22,7 +22,7 @@ */ /* -** Revision: 1.2 +** Revision: 1.4 */ import "riak.proto"; diff --git a/test/bucket_props_codec_eqc.erl b/test/bucket_props_codec_eqc.erl index da505fe0..ed235ad0 100644 --- a/test/bucket_props_codec_eqc.erl +++ b/test/bucket_props_codec_eqc.erl @@ -18,7 +18,7 @@ %% %% ------------------------------------------------------------------- -module(bucket_props_codec_eqc). - +-ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -31,7 +31,7 @@ bucket_codec_test_() -> [{"bucket properties encode decode", ?_test(begin - quickcheck(?QC_OUT(numtests(1000, prop_codec()))) + quickcheck(?QC_OUT(numtests(2000, prop_codec()))) end)}]. prop_codec() -> @@ -110,3 +110,5 @@ atom() -> linkfun() -> ?LET({M,F}, {atom(), atom()}, {linkfun, {modfun, M, F}}). + +-endif. From 98e331c6f169bfbe940837e27f968bd8111fa958 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 15 Feb 2013 15:51:50 -0600 Subject: [PATCH 08/10] Update README for message reorganization --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 945003a0..2dce1f99 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ single request. The response message will typically include a boolean RpbGetServerInfoReq -> RpbGetServerInfoResp RpbPingReq -> RpbPingResp - + RpbGetBucketReq -> RpbErrorResp | RpbGetBucketResp + RpbPutBucketReq -> RpbErrorResp | RpbPutBucketResp ### Riak KV Request/Response messages @@ -62,7 +63,6 @@ single request. The response message will typically include a boolean RpbDelReq -> RpbErrorResp | RpbDelResp RpbListBucketsReq -> RpbErrorResp | RpbListBucketsResp RpbListKeysReq -> RpbErrorResp | RpbListKeysResp{1,} - RpbGetBucketReq -> RpbErrorResp | RpbGetBucketResp RpbMapRedReq -> RpbMapRedResp{1,} RpbIndexReq -> RpbIndexResp From 0e0119e2afaff286821f7cb5961e2ac882eca546 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 27 Feb 2013 11:01:25 -0600 Subject: [PATCH 09/10] Make some shell helpers for the EQC property, idea from @russelldb --- test/bucket_props_codec_eqc.erl | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/test/bucket_props_codec_eqc.erl b/test/bucket_props_codec_eqc.erl index ed235ad0..cf6813ab 100644 --- a/test/bucket_props_codec_eqc.erl +++ b/test/bucket_props_codec_eqc.erl @@ -28,12 +28,18 @@ io:format(user, F, TL) end, P)). +%%==================================================================== +%% Eunit integration +%%==================================================================== bucket_codec_test_() -> - [{"bucket properties encode decode", + [{"bucket properties encode decode", ?_test(begin - quickcheck(?QC_OUT(numtests(2000, prop_codec()))) + eqc:quickcheck(?QC_OUT(eqc:numtests(2000, prop_codec()))) end)}]. +%%==================================================================== +%% Properties +%%==================================================================== prop_codec() -> ?FORALL(Props, sortuniq(list(bucket_prop())), ?WHENFAIL(begin @@ -47,6 +53,21 @@ prop_codec() -> Props =:= lists:sort(Props2) end)). +%%==================================================================== +%% Shell helpers +%%==================================================================== +qc() -> + qc(2000). + +qc(NumTests) -> + quickcheck(numtests(NumTests, prop_codec())). + +check() -> + eqc:check(prop_codec(), eqc:current_counterexample()). + +%%==================================================================== +%% Generators +%%==================================================================== bucket_prop() -> oneof([num(n_val), flag(allow_mult), From caa3d9ef90aa11c69cea42cbd29cf94ee1ce6613 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 27 Feb 2013 15:05:19 -0600 Subject: [PATCH 10/10] Fix try/catch pattern match when decoding modfuns. --- src/riak_pb_codec.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_pb_codec.erl b/src/riak_pb_codec.erl index f8e8a6c7..076508d2 100644 --- a/src/riak_pb_codec.erl +++ b/src/riak_pb_codec.erl @@ -357,7 +357,7 @@ decode_modfun(#rpbmodfun{module=Mod, function=Fun}=MF, _Prop) -> try {binary_to_existing_atom(Mod, latin1), binary_to_existing_atom(Fun, latin1)} catch - exit:{badarg,_} -> + error:badarg -> error_logger:warning_msg("Creating new atoms from protobuffs message! ~p", [MF]), {binary_to_atom(Mod, latin1), binary_to_atom(Fun, latin1)} end.