From 818ddb41ca24f80e267f3a72ff7e0ca579a03493 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 6 May 2013 10:47:41 -0500 Subject: [PATCH 1/5] Simplify option-list creation in riak_kv_pb_object. --- src/riak_kv_pb_object.erl | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/riak_kv_pb_object.erl b/src/riak_kv_pb_object.erl index 6c9d0c88b1..05a7b123b1 100644 --- a/src/riak_kv_pb_object.erl +++ b/src/riak_kv_pb_object.erl @@ -113,12 +113,9 @@ process(#rpbgetreq{bucket=B, key=K, r=R0, pr=PR0, notfound_ok=NFOk, timeout=Timeout}, #state{client=C} = State) -> R = decode_quorum(R0), PR = decode_quorum(PR0), - case C:get(B, K, make_option(deletedvclock, DeletedVClock) ++ - make_option(r, R) ++ - make_option(pr, PR) ++ - make_option(timeout, Timeout) ++ - make_option(notfound_ok, NFOk) ++ - make_option(basic_quorum, BQ)) of + case C:get(B, K, make_options([{deletedvclock, DeletedVClock}, + {r, R}, {pr, PR}, {timeout, Timeout}, + {notfound_ok, NFOk}, {basic_quorum, BQ}])) of {ok, O} -> case erlify_rpbvc(VClock) == riak_object:vclock(O) of true -> @@ -208,9 +205,8 @@ process(#rpbputreq{bucket=B, key=K, vclock=PbVC, content=RpbContent, _ -> [] end end, - case C:put(O, make_option(w, W) ++ make_option(dw, DW) ++ - make_option(pw, PW) ++ make_option(timeout, Timeout) ++ - Options) of + case C:put(O, make_options([{w, W}, {dw, DW}, {pw, PW}, + {timeout, Timeout}]) ++ Options) of ok when is_binary(ReturnKey) -> PutResp = #rpbputresp{key = ReturnKey}, {reply, PutResp, State}; @@ -250,13 +246,8 @@ process(#rpbdelreq{bucket=B, key=K, vclock=PbVc, PR = decode_quorum(PR0), RW = decode_quorum(RW0), - Options = make_option(r, R) ++ - make_option(w, W) ++ - make_option(rw, RW) ++ - make_option(pr, PR) ++ - make_option(pw, PW) ++ - make_option(dw, DW) ++ - make_option(timeout, Timeout), + Options = make_options([{r, R}, {w, W}, {rw, RW}, {pr, PR}, {pw, PW}, + {dw, DW}, {timeout, Timeout}]), Result = case PbVc of undefined -> C:delete(B, K, Options); @@ -293,6 +284,9 @@ update_pbvc(O0, PbVc) -> Vclock = erlify_rpbvc(PbVc), riak_object:set_vclock(O0, Vclock). +make_options(List) -> + lists:flatmap(fun({K,V}) -> make_option(K,V) end, List). + %% return a key/value tuple that we can ++ to other options so long as the %% value is not default or undefined -- those values are pulled from the %% bucket by the get/put FSMs. From 688bd5a14db5e56cc9fba52678a5c7b5fa5cdded Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 6 May 2013 11:02:05 -0500 Subject: [PATCH 2/5] Pass-through the asis put FSM flag over PBC. --- src/riak_kv_pb_object.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/riak_kv_pb_object.erl b/src/riak_kv_pb_object.erl index 05a7b123b1..97eb32c055 100644 --- a/src/riak_kv_pb_object.erl +++ b/src/riak_kv_pb_object.erl @@ -176,7 +176,7 @@ process(#rpbputreq{bucket=B, key=K, vclock=PbVC, process(#rpbputreq{bucket=B, key=K, vclock=PbVC, content=RpbContent, w=W0, dw=DW0, pw=PW0, return_body=ReturnBody, - return_head=ReturnHead, timeout=Timeout}, + return_head=ReturnHead, timeout=Timeout, asis=AsIs}, #state{client=C} = State) -> case K of @@ -206,7 +206,7 @@ process(#rpbputreq{bucket=B, key=K, vclock=PbVC, content=RpbContent, end end, case C:put(O, make_options([{w, W}, {dw, DW}, {pw, PW}, - {timeout, Timeout}]) ++ Options) of + {timeout, Timeout}, {asis, AsIs}]) ++ Options) of ok when is_binary(ReturnKey) -> PutResp = #rpbputresp{key = ReturnKey}, {reply, PutResp, State}; From 990de40d663e621f8c215fe84b4d8988d066db22 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 6 May 2013 15:13:53 -0500 Subject: [PATCH 3/5] Reorganize the malformed_request callback to simplify. --- src/riak_kv_wm_object.erl | 73 +++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/src/riak_kv_wm_object.erl b/src/riak_kv_wm_object.erl index 50377ed47b..0ae10385f4 100644 --- a/src/riak_kv_wm_object.erl +++ b/src/riak_kv_wm_object.erl @@ -226,44 +226,49 @@ allow_missing_post(RD, Ctx) -> %% at this time. malformed_request(RD, Ctx) when Ctx#ctx.method =:= 'POST' orelse Ctx#ctx.method =:= 'PUT' -> + malformed_request([fun malformed_content_type/2, + fun malformed_timeout_param/2, + fun malformed_rw_params/2, + fun malformed_link_headers/2, + fun malformed_index_headers/2], + RD, Ctx); +malformed_request(RD, Ctx) -> + malformed_request([fun malformed_timeout_param/2, + fun malformed_rw_params/2, + fun malformed_check_doc/2], RD, Ctx). + +%% @doc Given a list of 2-arity funs, threads through the request data +%% and context, returning as soon as a single fun discovers a +%% malformed request or halts. +%% -spec malformed_request([fun()], wrq:reqdata(), #ctx{}) -> {boolean() | {halt, non_neg_integer()}, wrq:reqdata(), #ctx{}}. +malformed_request([], RD, Ctx) -> + {false, RD, Ctx}; +malformed_request([H|T], RD, Ctx) -> + case H(RD, Ctx) of + {true, _, _} = Result -> Result; + {{halt,_}, _, _} = Halt -> Halt; + {false, RD1, Ctx1} -> + malformed_request(T, RD1, Ctx1) + end. + +%% @doc Detects whether the Content-Type header is missing on +%% PUT/POST. +malformed_content_type(RD, Ctx) -> case wrq:get_req_header("Content-Type", RD) of undefined -> {true, missing_content_type(RD), Ctx}; + _ -> {false, RD, Ctx} + end. + +%% @doc Detects whether fetching the requested object results in an +%% error. +malformed_check_doc(RD, Ctx) -> + DocCtx = ensure_doc(Ctx), + case DocCtx#ctx.doc of + {error, Reason} -> + handle_common_error(Reason, RD, DocCtx); _ -> - case malformed_timeout_param(RD, Ctx) of - Result={true, _, _} -> - Result; - {false, ToRD, ToCtx} -> - case malformed_rw_params(ToRD, ToCtx) of - Result={true, _, _} -> - Result; - {false, RWRD, RWCtx} -> - case malformed_link_headers(RWRD, RWCtx) of - Result = {true, _, _} -> - Result; - {false, RWLH, LHCtx} -> - malformed_index_headers(RWLH, LHCtx) - end - end - end - end; -malformed_request(RD, Ctx) -> - case malformed_timeout_param(RD, Ctx) of - Result={true, _, _} -> - Result; - {false, ToRD, ToCtx} -> - case malformed_rw_params(ToRD, ToCtx) of - Result = {true, _, _} -> - Result; - {false, ResRD, ResCtx} -> - DocCtx = ensure_doc(ResCtx), - case DocCtx#ctx.doc of - {error, Reason} -> - handle_common_error(Reason, ResRD, DocCtx); - _ -> - {false, ResRD, DocCtx} - end - end + {false, RD, DocCtx} end. %% @spec malformed_timeout_param(reqdata(), context()) -> From 4d821d365462d1fa76247b90fa4fb59b16a35ade Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 6 May 2013 15:14:29 -0500 Subject: [PATCH 4/5] Whitespace cleanup in riak_kv_wm_object --- src/riak_kv_wm_object.erl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/riak_kv_wm_object.erl b/src/riak_kv_wm_object.erl index 0ae10385f4..510f3243cc 100644 --- a/src/riak_kv_wm_object.erl +++ b/src/riak_kv_wm_object.erl @@ -280,7 +280,7 @@ malformed_timeout_param(RD, Ctx) -> case wrq:get_qs_value("timeout", none, RD) of none -> {false, RD, Ctx}; - TimeoutStr -> + TimeoutStr -> try Timeout = list_to_integer(TimeoutStr), {false, RD, Ctx#ctx{timeout=Timeout}} @@ -290,7 +290,7 @@ malformed_timeout_param(RD, Ctx) -> wrq:append_to_resp_body(io_lib:format("Bad timeout " "value ~p~n", [TimeoutStr]), - wrq:set_resp_header(?HEAD_CTYPE, + wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), Ctx} end @@ -397,7 +397,7 @@ malformed_link_headers(RD, Ctx) -> %% @spec malformed_index_headers(reqdata(), context()) -> %% {boolean(), reqdata(), context()} %% -%% @doc Check that the Index headers (HTTP headers prefixed with index_") +%% @doc Check that the Index headers (HTTP headers prefixed with index_") %% are valid. Store the parsed headers in context() if valid, %% or print an error in reqdata() if not. %% An index field should be of the form "index_fieldname_type" @@ -751,7 +751,7 @@ multiple_choices(RD, Ctx) -> %% property "rel=container". The rest of the links will be %% constructed from the links of the document. produce_doc_body(RD, Ctx) -> - Prefix = Ctx#ctx.prefix, + Prefix = Ctx#ctx.prefix, Bucket = Ctx#ctx.bucket, APIVersion = Ctx#ctx.api_version, case select_doc(Ctx) of @@ -763,7 +763,7 @@ produce_doc_body(RD, Ctx) -> end, Links2 = riak_kv_wm_utils:format_links([{Bucket, "up"}|Links1], Prefix, APIVersion), LinkRD = wrq:merge_resp_headers(Links2, RD), - + %% Add user metadata to response... UserMetaRD = case dict:find(?MD_USERMETA, MD) of {ok, UserMeta} -> @@ -896,7 +896,7 @@ ensure_doc(Ctx) -> Ctx. delete_resource(RD, Ctx=#ctx{bucket=B, key=K, client=C}) -> Options = make_options([], Ctx), Result = case wrq:get_req_header(?HEAD_VCLOCK, RD) of - undefined -> + undefined -> C:delete(B,K,Options); _ -> C:delete_vclock(B,K,decode_vclock_header(RD),Options) @@ -937,7 +937,7 @@ last_modified(RD, Ctx) -> multiple_choices -> {ok, Doc} = Ctx#ctx.doc, LMDates = [ normalize_last_modified(MD) || - MD <- riak_object:get_metadatas(Doc) ], + MD <- riak_object:get_metadatas(Doc) ], {lists:max(LMDates), RD, Ctx} end. @@ -962,7 +962,7 @@ get_link_heads(RD, Ctx) -> Bucket = Ctx#ctx.bucket, %% Get a list of link headers... - LinkHeaders1 = + LinkHeaders1 = case wrq:get_req_header(?HEAD_LINK, RD) of undefined -> []; Heads -> string:tokens(Heads, ",") @@ -970,7 +970,7 @@ get_link_heads(RD, Ctx) -> %% Decode the link headers. Throw an exception if we can't %% properly parse any of the headers... - {BucketLinks, KeyLinks} = + {BucketLinks, KeyLinks} = case APIVersion of 1 -> {ok, BucketRegex} = re:compile("; ?rel=\"([^\"]+)\""), @@ -986,12 +986,12 @@ get_link_heads(RD, Ctx) -> %% bucket... IsValid = (BucketLinks == []) orelse (BucketLinks == [{Bucket, <<"up">>}]), case IsValid of - true -> + true -> KeyLinks; false -> throw({invalid_link_headers, LinkHeaders1}) end. - + %% Run each LinkHeader string() through the BucketRegex and %% KeyRegex. Return {BucketLinks, KeyLinks}. extract_links(LinkHeaders, BucketRegex, KeyRegex) -> @@ -1119,9 +1119,9 @@ handle_common_error(Reason, RD, Ctx) -> end. make_options(Prev, Ctx) -> - NewOpts0 = [{rw, Ctx#ctx.rw}, {r, Ctx#ctx.r}, {w, Ctx#ctx.w}, - {pr, Ctx#ctx.pr}, {pw, Ctx#ctx.pw}, {dw, Ctx#ctx.dw}, + NewOpts0 = [{rw, Ctx#ctx.rw}, {r, Ctx#ctx.r}, {w, Ctx#ctx.w}, + {pr, Ctx#ctx.pr}, {pw, Ctx#ctx.pw}, {dw, Ctx#ctx.dw}, {timeout, Ctx#ctx.timeout}], - NewOpts = [ {Opt, Val} || {Opt, Val} <- NewOpts0, + NewOpts = [ {Opt, Val} || {Opt, Val} <- NewOpts0, Val /= undefined, Val /= default ], Prev ++ NewOpts. From 4e2940752dfc6edf04779ab56f575222d4d02c4e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 6 May 2013 15:18:08 -0500 Subject: [PATCH 5/5] Expose asis flag to HTTP --- src/riak_kv_wm_object.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/riak_kv_wm_object.erl b/src/riak_kv_wm_object.erl index 510f3243cc..9694470b30 100644 --- a/src/riak_kv_wm_object.erl +++ b/src/riak_kv_wm_object.erl @@ -134,6 +134,7 @@ pw, %% integer() - number of primary nodes required in preflist on write basic_quorum, %% boolean() - whether to use basic_quorum notfound_ok, %% boolean() - whether to treat notfounds as successes + asis, %% boolean() - whether to send the put without modifying the vclock prefix, %% string() - prefix for resource uris riak, %% local | {node(), atom()} - params for riak client doc, %% {ok, riak_object()}|{error, term()} - the object found @@ -314,7 +315,8 @@ malformed_rw_params(RD, Ctx) -> lists:foldl(fun malformed_boolean_param/2, Res, [{#ctx.basic_quorum, "basic_quorum", "default"}, - {#ctx.notfound_ok, "notfound_ok", "default"}]). + {#ctx.notfound_ok, "notfound_ok", "default"}, + {#ctx.asis, "asis", "false"}]). %% @spec malformed_rw_param({Idx::integer(), Name::string(), Default::string()}, %% {boolean(), reqdata(), context()}) -> @@ -1121,7 +1123,7 @@ handle_common_error(Reason, RD, Ctx) -> make_options(Prev, Ctx) -> NewOpts0 = [{rw, Ctx#ctx.rw}, {r, Ctx#ctx.r}, {w, Ctx#ctx.w}, {pr, Ctx#ctx.pr}, {pw, Ctx#ctx.pw}, {dw, Ctx#ctx.dw}, - {timeout, Ctx#ctx.timeout}], + {timeout, Ctx#ctx.timeout}, {asis, Ctx#ctx.asis}], NewOpts = [ {Opt, Val} || {Opt, Val} <- NewOpts0, Val /= undefined, Val /= default ], Prev ++ NewOpts.