diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..806862f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +log/ +misc/ +dev/ +_build/ +rel/pkg/ +rel/stanchion/ diff --git a/.gitignore b/.gitignore index fb01f02..4efb0dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,9 @@ .eunit/* -deps/* -priv/* +_build/ rel/stanchion -ebin/*.beam -ebin/*.app -doc dev /package/ log/ *~ /log -riak_test/bin/stanchion* \ No newline at end of file +riak_test/bin/stanchion* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7067a48 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM erlang:22 AS compile-image + +EXPOSE 8085 + +WORKDIR /usr/src/stanchion +COPY . /usr/src/stanchion + +# When running in a docker container, ideally we would want our app to +# be configurable via environment variables (option --env-file to +# docker run). For that reason, We use a pared-down, cuttlefish-less +# rebar.config. Configuration from environment now becomes possible, +# via rebar's own method of generating sys.config from +# /sys.config.src. +RUN make rel-docker + +FROM debian:bullseye AS runtime-image + +COPY --from=compile-image /usr/src/stanchion/rel/stanchion /opt/stanchion + +CMD /opt/stanchion/bin/stanchion foreground diff --git a/Makefile b/Makefile index 6570c4f..6f339ee 100644 --- a/Makefile +++ b/Makefile @@ -1,107 +1,134 @@ REPO ?= stanchion - -PKG_REVISION ?= $(shell git describe --tags) -PKG_VERSION ?= $(shell git describe --tags | tr - .) -PKG_ID = stanchion-$(PKG_VERSION) +HEAD_REVISION ?= $(shell git describe --tags --exact-match HEAD 2>/dev/null) +PKG_REVISION ?= $(shell git describe --tags 2>/dev/null) PKG_BUILD = 1 BASE_DIR = $(shell pwd) -ERLANG_BIN = $(shell dirname $(shell which erl)) -REBAR ?= $(BASE_DIR)/rebar +ERLANG_BIN = $(shell dirname $(shell which erl 2>/dev/null) 2>/dev/null) +OTP_VER = $(shell erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell) +REBAR ?= $(BASE_DIR)/rebar3 OVERLAY_VARS ?= .PHONY: rel deps test -all: deps compile +all: compile -compile: deps - @(./rebar compile) +compile: + @($(REBAR) compile) deps: - @./rebar get-deps + @$(REBAR) upgrade --all clean: - @./rebar clean - -distclean: clean - @./rebar delete-deps - @rm -rf $(PKG_ID).tar.gz + @$(REBAR) clean -parity-test: - @python test/prototype_parity.py -v +distclean: clean devclean relclean + @rm -rf _build ## ## Release targets ## -rel: deps compile - @./rebar compile - @./rebar skip_deps=true generate $(OVERLAY_VARS) +rel: compile + @$(REBAR) as rel release + @cp -a _build/rel/rel/stanchion rel/ + +rel-rpm: compile relclean + @$(REBAR) as rpm release + @cp -a _build/rpm/rel/stanchion rel/ + +rel-deb: compile relclean + @$(REBAR) as deb release + @cp -a _build/deb/rel/stanchion rel/ + +rel-fbsdng: compile relclean + @$(REBAR) as fbsdng release + @cp -a _build/fbsdng/rel/stanchion rel/ + +rel-alpine: compile relclean + @$(REBAR) as alpine release + @(cd _build/alpine/rel/stanchion/usr/bin && mv stanchion.nosu stanchion) + @cp -a _build/alpine/rel/stanchion rel/ + +rel-osx: compile relclean + @$(REBAR) as osx release + @cp -a _build/osx/rel/stanchion rel/ + +rel-docker: compile relclean + @REBAR_CONFIG=rebar.docker.config $(REBAR) release + @cp -a _build/default/rel/stanchion rel/ relclean: - rm -rf rel/stanchion + @rm -rf _build/rel rel/stanchion + +test: + @$(REBAR) eunit + @$(REBAR) dialyzer ## ## Developer targets ## -stage : rel - $(foreach dep,$(wildcard deps/*), rm -rf rel/stanchion/lib/$(shell basename $(dep))-* && ln -sf $(abspath $(dep)) rel/stanchion/lib;) - $(foreach app,$(wildcard apps/*), rm -rf rel/stanchion/lib/$(shell basename $(app))-* && ln -sf $(abspath $(app)) rel/stanchion/lib;) - devrel: all - mkdir -p dev - @./rebar skip_deps=true generate target_dir=../dev/$(REPO) \ - overlay_vars=dev_vars.config + @mkdir -p dev + @$(REBAR) as rel release -o dev --overlay_vars rel/dev_vars.config stagedevrel: devrel - $(foreach app,$(wildcard apps/*), rm -rf dev/$(REPO)/lib/$(shell basename $(app))* && ln -sf $(abspath $(app)) dev/$(REPO)/lib;) - $(foreach dep,$(wildcard deps/*), rm -rf dev/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$(REPO)/lib;) devclean: clean rm -rf dev ## -## Doc targets +## Version and naming variables for distribution and packaging ## -orgs: orgs-doc orgs-README - -orgs-doc: - @emacs -l orgbatch.el -batch --eval="(riak-export-doc-dir \"doc\" 'html)" - -orgs-README: - @emacs -l orgbatch.el -batch --eval="(riak-export-doc-file \"README.org\" 'ascii)" - @mv README.txt README -DIALYZER_APPS = kernel stdlib sasl erts ssl tools os_mon runtime_tools crypto inets \ - xmerl webtool eunit syntax_tools compiler -PLT ?= $(HOME)/.stanchion_dialyzer_plt +# Tag from git with style -- +# Ex: When on a tag: riak-1.0.3 (no commits since tag) +# For most normal Commits: riak-1.1.0pre1-27-g1170096 +# Last tag: riak-1.1.0pre1 +# Commits since tag: 27 +# Hash of commit: g1170096 +REPO_TAG := $(shell git describe --tags) + +# Split off repo name +# Changes to 1.0.3 or 1.1.0pre1-27-g1170096 from example above +REVISION = $(shell echo $(REPO_TAG) | sed -e 's/^$(REPO)-//') + +# Primary version identifier, strip off commmit information +# Changes to 1.0.3 or 1.1.0pre1 from example above +MAJOR_VERSION ?= $(shell echo $(REVISION) | sed -e 's/\([0-9.]*\)-.*/\1/') + +# Name resulting directory & tar file based on current status of the git tag +# If it is a tagged release (PKG_VERSION == MAJOR_VERSION), use the toplevel +# tag as the package name, otherwise generate a unique hash of all the +# dependencies revisions to make the package name unique. +# This enables the toplevel repository package to change names +# when underlying dependencies change. +NAME_HASH = $(shell git hash-object distdir/$(CLONEDIR)/$(MANIFEST_FILE) 2>/dev/null | cut -c 1-8) +PKG_ID := "$(REPO_TAG)-OTP$(OTP_VER)" ## ## Packaging targets ## + +# Yes another variable, this one is repo-- +PKG_VERSION = $(shell echo $(PKG_ID) | sed -e 's/^$(REPO)-//') + +package: + mkdir -p rel/pkg/out/stanchion-$(PKG_ID) + git archive --format=tar HEAD | gzip >rel/pkg/out/$(PKG_ID).tar.gz + $(MAKE) -f rel/pkg/Makefile + +packageclean: + rm -rf rel/pkg/out/* + + .PHONY: package export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE -package.src: deps - mkdir -p package - rm -rf package/$(PKG_ID) - git archive --format=tar --prefix=$(PKG_ID)/ $(PKG_REVISION)| (cd package && tar -xf -) - ${MAKE} -C package/$(PKG_ID) deps - mkdir -p package/$(PKG_ID)/priv - git --git-dir=.git describe --tags >package/$(PKG_ID)/priv/vsn.git - for dep in package/$(PKG_ID)/deps/*; do \ - echo "Processing dep: $${dep}"; \ - mkdir -p $${dep}/priv; \ - git --git-dir=$${dep}/.git describe --tags >$${dep}/priv/vsn.git; \ - done - find package/$(PKG_ID) -depth -name ".git" -exec rm -rf {} \; - tar -C package -czf package/$(PKG_ID).tar.gz $(PKG_ID) - -dist: package.src - cp package/$(PKG_ID).tar.gz . - -package: package.src - ${MAKE} -C package -f $(PKG_ID)/deps/node_package/Makefile - -pkgclean: distclean - rm -rf package - -include tools.mk +# Package up a devrel to save time later rebuilding it +pkg-devrel: devrel + echo -n $(PKG_REVISION) > VERSION + tar -czf $(PKG_ID)-devrel.tar.gz dev/ VERSION + rm -rf VERSION + +pkg-rel: rel + tar -czf $(PKG_ID)-rel.tar.gz -C rel/ . diff --git a/RELEASE-NOTES.org b/RELEASE-NOTES.org index 9800a6a..68d4b87 100644 --- a/RELEASE-NOTES.org +++ b/RELEASE-NOTES.org @@ -1,5 +1,38 @@ +* Stanchion 3.0.0 Release Notes +** Changes +- This release is to be used with Riak CS 3.0.0. It requires + OTP-22 and rebar3, with no changes that would affect the user + otherwise. +- Packages (rpm and deb) now rely on systemd, currently available for + Centos 7 and 8, Debian 8 and 11, FreeBSD 13 and OSX 14. +- A Dockerfile. +* Stanchion 2.1.2 Release Notes +** Additions +- Update to node_package 3.0.0 +* Stanchion 2.1.1 Release Notes +** Bugs Fixed +- Remove admin.secret from configuration (riak_cs/#1274) +* Stanchion 2.1.0 Release Notes +** Additions +- Add statistics reporting +- Add exometer_core 1.2 +- Update to cuttlefish 2.0.4 +- Update to eper 0.92-basho1 +- Update to lager 2.2.0 +- Update to lager_syslog 2.1.1 +- Update to node_package 2.0.3 +- Update to riakc 2.1.1 +* Stanchion 2.0.0 Release Notes +** Additions +- Change to cuttlefish-based configuration +- Add cuttlefish 2.0.1 +- Update to eper 0.78 +- Update to node_package 2.0.1 +- Update to riakc 2.1.0 +- Update to webmachine 1.10.8 +- Update to Erlang/OTP R16B02 * Stanchion 1.5.0 Release Notes -** Addisions +** Additions - Update to lager 2.0.3 - Compiles with R16B0x (Releases still by R15B01) ** Bugs Fixed diff --git a/apps/stanchion/ebin b/apps/stanchion/ebin deleted file mode 120000 index fe2cfa5..0000000 --- a/apps/stanchion/ebin +++ /dev/null @@ -1 +0,0 @@ -../../ebin \ No newline at end of file diff --git a/apps/stanchion/include b/apps/stanchion/include deleted file mode 120000 index 3611dd2..0000000 --- a/apps/stanchion/include +++ /dev/null @@ -1 +0,0 @@ -../../include/ \ No newline at end of file diff --git a/apps/stanchion/include/stanchion.hrl b/apps/stanchion/include/stanchion.hrl new file mode 100644 index 0000000..4fff460 --- /dev/null +++ b/apps/stanchion/include/stanchion.hrl @@ -0,0 +1,34 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-record(context, {auth_bypass :: boolean(), + bucket :: undefined | binary(), + owner_id :: undefined | all | string()}). + +-define(TURNAROUND_TIME(Call), + begin + StartTime_____tat = os:timestamp(), + Result_____tat = (Call), + EndTime_____tat = os:timestamp(), + {Result_____tat, + timer:now_diff(EndTime_____tat, + StartTime_____tat)} + end). diff --git a/apps/stanchion/src b/apps/stanchion/src deleted file mode 120000 index 929cb3d..0000000 --- a/apps/stanchion/src +++ /dev/null @@ -1 +0,0 @@ -../../src \ No newline at end of file diff --git a/src/stanchion.app.src b/apps/stanchion/src/stanchion.app.src similarity index 57% rename from src/stanchion.app.src rename to apps/stanchion/src/stanchion.app.src index 888969a..18bdcdf 100644 --- a/src/stanchion.app.src +++ b/apps/stanchion/src/stanchion.app.src @@ -2,6 +2,10 @@ {application, stanchion, [ {description, "stanchion"}, + %% 'git' may introduce a '+' character (when building not from a + %% tag), and that plus sign will not go down well with freebsd pkg + %% ('+' is not allowed in +MANIFEST file). Hence we produce a + %% version of vsn that doesn't have it. {vsn, git}, {modules, []}, {registered, []}, @@ -12,7 +16,10 @@ crypto, mochiweb, webmachine, - lager + exometer_core, + lager, + riakc, + rcs_common ]}, {mod, { stanchion_app, []}}, {env, [ diff --git a/src/stanchion_acl_utils.erl b/apps/stanchion/src/stanchion_acl_utils.erl similarity index 97% rename from src/stanchion_acl_utils.erl rename to apps/stanchion/src/stanchion_acl_utils.erl index cf47ff9..125b63c 100644 --- a/src/stanchion_acl_utils.erl +++ b/apps/stanchion/src/stanchion_acl_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,13 +23,11 @@ -module(stanchion_acl_utils). --include("stanchion.hrl"). +-include_lib("rcs_common/include/rcs_common_manifest.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -ifdef(TEST). - -include_lib("eunit/include/eunit.hrl"). - -endif. %% Public API @@ -43,7 +42,7 @@ %% @doc Construct an acl. The structure is the same for buckets %% and objects. --spec acl(string(), string(), string(), [acl_grant()], erlang:timestamp()) -> acl2(). +-spec acl(string(), string(), string(), [acl_grant()], erlang:timestamp()) -> acl(). acl(DisplayName, CanonicalId, KeyId, Grants, CreationTime) -> OwnerData = {DisplayName, CanonicalId, KeyId}, ?ACL{owner=OwnerData, @@ -52,7 +51,7 @@ acl(DisplayName, CanonicalId, KeyId, Grants, CreationTime) -> %% @doc Convert a set of JSON terms representing an ACL into %% an internal representation. --spec acl_from_json(term()) -> acl2(). +-spec acl_from_json(term()) -> acl(). acl_from_json({struct, Json}) -> process_acl_contents(Json, ?ACL{}); acl_from_json(Json) -> @@ -123,7 +122,7 @@ permissions_to_json_term(Perms) -> [list_to_binary(atom_to_list(Perm)) || Perm <- Perms]. %% @doc Process the top-level elements of the --spec process_acl_contents([term()], acl()) -> acl2(). +-spec process_acl_contents([term()], acl()) -> acl(). process_acl_contents([], Acl) -> Acl; process_acl_contents([{Name, Value} | RestObjects], Acl) -> @@ -144,7 +143,7 @@ process_acl_contents([{Name, Value} | RestObjects], Acl) -> process_acl_contents(RestObjects, UpdAcl). %% @doc Process an JSON element containing acl owner information. --spec process_owner([term()], acl()) -> acl2(). +-spec process_owner([term()], acl()) -> acl(). process_owner([], Acl) -> Acl; process_owner([{Name, Value} | RestObjects], Acl) -> @@ -169,7 +168,7 @@ process_owner([{Name, Value} | RestObjects], Acl) -> process_owner(RestObjects, Acl?ACL{owner=UpdOwner}). %% @doc Process an JSON element containing the grants for the acl. --spec process_grants([term()], acl()) -> acl2(). +-spec process_grants([term()], acl()) -> acl(). process_grants([], Acl) -> Acl; process_grants([{_, Value} | RestObjects], Acl) -> @@ -261,7 +260,7 @@ process_creation_time([{Name, Value} | RestObjects], CreationTime) -> -ifdef(TEST). acl_from_json_test() -> - CreationTime = erlang:now(), + CreationTime = erlang:timestamp(), {AclMegaSecs, AclSecs, AclMicroSecs} = CreationTime, JsonTerm = [{<<"version">>,1}, {<<"owner">>, @@ -297,7 +296,7 @@ acl_from_json_test() -> ?assertEqual(ExpectedAcl, Acl). acl_to_json_term_test() -> - CreationTime = erlang:now(), + CreationTime = erlang:timestamp(), Acl = acl("tester1", "TESTID1", "TESTKEYID1", @@ -340,7 +339,7 @@ owner_to_json_term_test() -> ?assertEqual(ExpectedTerm, JsonTerm). grants_to_json_term_test() -> - CreationTime = erlang:now(), + CreationTime = erlang:timestamp(), Acl = acl("tester1", "TESTID1", "TESTKEYID1", diff --git a/apps/stanchion/src/stanchion_app.erl b/apps/stanchion/src/stanchion_app.erl new file mode 100644 index 0000000..2b0ac06 --- /dev/null +++ b/apps/stanchion/src/stanchion_app.erl @@ -0,0 +1,102 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Callbacks for the stanchion application. + +-module(stanchion_app). + +-behaviour(application). + +-include_lib("rcs_common/include/rcs_common_moss.hrl"). + +%% application API +-export([start/2, + stop/1]). + + +-type start_type() :: normal | {takeover, node()} | {failover, node()}. +-type start_args() :: term(). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc application start callback for stanchion. +-spec start(start_type(), start_args()) -> {ok, pid()} | + {error, term()}. +start(_Type, _StartArgs) -> + case stanchion_utils:riak_connection() of + {ok, Pid} -> + try + case check_admin_creds(Pid) of + ok -> + stanchion_sup:start_link(); + Error -> + Error + end + after + stanchion_utils:close_riak_connection(Pid) + end; + {error, Reason} -> + lager:error("Couldn't connect to Riak: ~p", [Reason]), + {error, Reason} + end. + +%% @doc application stop callback for stanchion. +-spec stop(term()) -> ok. +stop(_State) -> + ok. + +check_admin_creds(Pid) -> + case application:get_env(stanchion, admin_key) of + {ok, "admin-key"} -> + lager:warning("admin.key is defined as default. Please create" + " admin user and configure it.", []), + application:set_env(stanchion, admin_secret, "admin-secret"); + {ok, KeyId} -> + case application:get_env(stanchion, admin_secret) of + {ok, _} -> + lager:warning("admin.secret is ignored."); + _ -> + ok + end, + StrongOpts = [{r, quorum}, {pr, one}, {notfound_ok, false}], + case riakc_pb_socket:get(Pid, ?USER_BUCKET, KeyId, StrongOpts) of + {ok, Obj} -> + case stanchion_utils:from_riakc_obj(Obj, false) of + {ok, {User, _}} -> + Secret = User?RCS_USER.key_secret, + application:set_env(stanchion, admin_secret, Secret); + Error -> + Error + end; + {error, notfound} -> + lager:error("admin.key defined in stanchion.conf was not found." + "Please create it."), + {error, admin_not_configured}; + Error -> + lager:error("Error loading administrator configuration: ~p", + [Error]), + Error + end; + Error -> + Error + end. diff --git a/src/stanchion_auth.erl b/apps/stanchion/src/stanchion_auth.erl similarity index 98% rename from src/stanchion_auth.erl rename to apps/stanchion/src/stanchion_auth.erl index 67e3fd6..3d9d41a 100644 --- a/src/stanchion_auth.erl +++ b/apps/stanchion/src/stanchion_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_blockall_auth.erl b/apps/stanchion/src/stanchion_blockall_auth.erl similarity index 94% rename from src/stanchion_blockall_auth.erl rename to apps/stanchion/src/stanchion_blockall_auth.erl index 544ea9d..cab0af3 100644 --- a/src/stanchion_blockall_auth.erl +++ b/apps/stanchion/src/stanchion_blockall_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_console.erl b/apps/stanchion/src/stanchion_console.erl similarity index 84% rename from src/stanchion_console.erl rename to apps/stanchion/src/stanchion_console.erl index cb68c37..62f2cfb 100644 --- a/src/stanchion_console.erl +++ b/apps/stanchion/src/stanchion_console.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,11 +23,18 @@ -module(stanchion_console). --export([status/1]). +-export([version/1, + status/1]). %% =================================================================== %% Public API %% =================================================================== + +version(_) -> + {ok, Vsn} = application:get_env(stanchion, stanchion_version), + io:format("version : ~p~n", [Vsn]), + ok. + status(_) -> Stats = stanchion_stats:get_stats(), [io:format("~s : ~p~n", [Key,Value]) || {Key,Value} <- Stats], diff --git a/apps/stanchion/src/stanchion_multipart.erl b/apps/stanchion/src/stanchion_multipart.erl new file mode 100644 index 0000000..e35ab17 --- /dev/null +++ b/apps/stanchion/src/stanchion_multipart.erl @@ -0,0 +1,79 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_multipart). + +-export([check_no_multipart_uploads/2]). + +-include("stanchion.hrl"). +-include_lib("rcs_common/include/rcs_common_manifest.hrl"). + + +check_no_multipart_uploads(Bucket, RiakPid) -> + HashBucket = stanchion_utils:to_bucket_name(objects, Bucket), + + {{ok, Keys}, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:list_keys(RiakPid, HashBucket)), + stanchion_stats:update([riakc, list_all_manifest_keys], TAT), + + %% check all up + lists:all(fun(VKey) -> + {Key, Vsn} = rcs_common_manifest:decompose_versioned_key(VKey), + GetResult = stanchion_utils:get_manifests_raw(RiakPid, Bucket, Key, Vsn), + has_no_upload(GetResult) + end, Keys). + +has_no_upload({ok, Obj}) -> + Manifests = manifests_from_riak_object(Obj), + lists:all(fun({_UUID,Manifest}) -> + case Manifest?MANIFEST.state of + writing -> + %% if this is mp => false + not proplists:is_defined(multipart, Manifest?MANIFEST.props); + _ -> + true + end + end, Manifests); +has_no_upload({error, notfound}) -> true. + +-spec manifests_from_riak_object(riakc_obj:riakc_obj()) -> orddict:orddict(). +manifests_from_riak_object(RiakObject) -> + %% For example, riak_cs_manifest_fsm:get_and_update/4 may wish to + %% update the #riakc_obj without a roundtrip to Riak first. So we + %% need to see what the latest + Contents = try + %% get_update_value will return the updatevalue or + %% a single old original value. + [{riakc_obj:get_update_metadata(RiakObject), + riakc_obj:get_update_value(RiakObject)}] + catch throw:_ -> + %% Original value had many contents + riakc_obj:get_contents(RiakObject) + end, + DecodedSiblings = [binary_to_term(V) || + {_, V}=Content <- Contents, + not stanchion_utils:has_tombstone(Content)], + + %% Upgrade the manifests to be the latest erlang + %% record version + Upgraded = rcs_common_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), + + %% resolve the siblings + rcs_common_manifest_resolution:resolve(Upgraded). diff --git a/src/stanchion_passthru_auth.erl b/apps/stanchion/src/stanchion_passthru_auth.erl similarity index 94% rename from src/stanchion_passthru_auth.erl rename to apps/stanchion/src/stanchion_passthru_auth.erl index affcc20..6665b3b 100644 --- a/src/stanchion_passthru_auth.erl +++ b/apps/stanchion/src/stanchion_passthru_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_response.erl b/apps/stanchion/src/stanchion_response.erl similarity index 98% rename from src/stanchion_response.erl rename to apps/stanchion/src/stanchion_response.erl index bfb3d1a..a99910f 100644 --- a/src/stanchion_response.erl +++ b/apps/stanchion/src/stanchion_response.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,8 +25,8 @@ respond/4, error_response/5, list_buckets_response/3]). + -define(xml_prolog, ""). --include("stanchion.hrl"). error_message(invalid_access_key_id) -> "The AWS Access Key Id you provided does not exist in our records."; diff --git a/src/stanchion_server.erl b/apps/stanchion/src/stanchion_server.erl similarity index 91% rename from src/stanchion_server.erl rename to apps/stanchion/src/stanchion_server.erl index 5717ec3..755b3dc 100644 --- a/src/stanchion_server.erl +++ b/apps/stanchion/src/stanchion_server.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -41,6 +42,7 @@ delete_bucket/2, set_bucket_acl/2, set_bucket_policy/2, + set_bucket_versioning/2, delete_bucket_policy/2, stop/1, update_user/2, @@ -54,8 +56,8 @@ terminate/2, code_change/3]). --record(state, {riak_ip :: string(), - riak_port :: pos_integer()}). +-record(state, {riak_ip :: undefined | string(), + riak_port :: undefined | pos_integer()}). -type state() :: #state{}. %% This ?TURNAROUND_TIME has another ?TURNAROUND_TIME at gen_server @@ -122,6 +124,14 @@ set_bucket_policy(Bucket, FieldList) -> {set_policy, Bucket, FieldList}, infinity)). +%% @doc Set the versioning option for a bucket +-spec set_bucket_versioning(binary(), term()) -> ok | {error, term()}. +set_bucket_versioning(Bucket, FieldList) -> + ?MEASURE([bucket, set_versioning], + gen_server:call(?MODULE, + {set_versioning, Bucket, FieldList}, + infinity)). + %% @doc delete the policy for a bucket -spec delete_bucket_policy(binary(), binary()) -> ok | {error, term()}. delete_bucket_policy(Bucket, RequesterId) -> @@ -159,7 +169,7 @@ msgq_len() -> init([]) -> {ok, #state{}}; init(test) -> - {ok, #state{}}. + {ok, #state{}}. %% @doc Handle synchronous commands issued via exported functions. -spec handle_call(term(), {pid(), term()}, state()) -> @@ -194,6 +204,11 @@ handle_call({set_policy, Bucket, FieldList}, State=#state{}) -> Result = ?TURNAROUND_TIME(stanchion_utils:set_bucket_policy(Bucket, FieldList)), {reply, Result, State}; +handle_call({set_versioning, Bucket, FieldList}, + _From, + State=#state{}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:set_bucket_versioning(Bucket, FieldList)), + {reply, Result, State}; handle_call({delete_policy, Bucket, RequesterId}, _From, State=#state{}) -> diff --git a/src/stanchion_server_sup.erl b/apps/stanchion/src/stanchion_server_sup.erl similarity index 74% rename from src/stanchion_server_sup.erl rename to apps/stanchion/src/stanchion_server_sup.erl index dc96f40..e2bcae5 100644 --- a/src/stanchion_server_sup.erl +++ b/apps/stanchion/src/stanchion_server_sup.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -43,23 +44,17 @@ start_link() -> %% @doc Initialize this supervisor. This is a `one_for_one', %% whose child spec is for starting a `stanchion_server' process. --spec init([]) -> {ok, {{supervisor:strategy(), - pos_integer(), - pos_integer()}, - [supervisor:child_spec()]}}. +-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. init([]) -> - RestartStrategy = one_for_one, - MaxRestarts = 1000, - MaxSecondsBetweenRestarts = 3600, - - SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, - - Restart = permanent, - Shutdown = 2000, - Type = worker, - - ServerSpec = {undefined, - {stanchion_server, start_link, []}, - Restart, Shutdown, Type, [stanchion_server]}, + SupFlags = #{strategy => one_for_one, + intensity => 1000, + period => 3600}, + + ServerSpec = #{id => stanchion_server, + start => {stanchion_server, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [stanchion_server]}, {ok, {SupFlags, [ServerSpec]}}. diff --git a/src/stanchion_stats.erl b/apps/stanchion/src/stanchion_stats.erl similarity index 97% rename from src/stanchion_stats.erl rename to apps/stanchion/src/stanchion_stats.erl index 758ca3f..8348a81 100644 --- a/src/stanchion_stats.erl +++ b/apps/stanchion/src/stanchion_stats.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -42,6 +43,7 @@ duration_metrics() -> [bucket, put_acl], [bucket, set_policy], [bucket, delete_policy], + [bucket, set_versioning], [user, create], [user, update], @@ -234,10 +236,10 @@ stats_metric_test() -> end || {Key, Type, Options, _} <- all_metrics()]. stats_test_() -> - Apps = [setup, compiler, syntax_tools, goldrush, lager, exometer_core], + Apps = [setup, compiler, syntax_tools, exometer_core], {setup, fun() -> - [ok = application:start(App) || App <- Apps], + [application:ensure_all_started(App) || App <- Apps], ok = stanchion_stats:init() end, fun(_) -> diff --git a/src/stanchion_sup.erl b/apps/stanchion/src/stanchion_sup.erl similarity index 55% rename from src/stanchion_sup.erl rename to apps/stanchion/src/stanchion_sup.erl index 1e32e74..3373761 100644 --- a/src/stanchion_sup.erl +++ b/apps/stanchion/src/stanchion_sup.erl @@ -43,10 +43,8 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). %% @doc supervisor callback. --spec init([]) -> {ok, {{supervisor:strategy(), - integer(), - integer()}, - [supervisor:child_spec()]}}. +-spec init([]) -> {ok, {supervisor:sup_flags(), + [supervisor:child_spec()]}} | ignore. init([]) -> {Ip, Port} = case application:get_env(stanchion, host) of {ok, {_, _} = HostPort} -> HostPort; @@ -54,28 +52,41 @@ init([]) -> end, stanchion_stats:init(), - + %% Hide any bags from user-facing parts. + case application:get_env(stanchion, supercluster_members) of + undefined -> ok; + {ok, Bags} -> application:set_env(stanchion, bags, Bags) + end, + %% Create child specifications WebConfig1 = [ - {dispatch, stanchion_web:dispatch_table()}, - {ip, Ip}, - {port, Port}, - {nodelay, true}, - {log_dir, "log"}, - %% {rewrite_module, stanchion_wm_rewrite}, - {error_handler, stanchion_wm_error_handler}], - case application:get_env(stanchion, ssl) of - {ok, SSLOpts} -> - WebConfig = WebConfig1 ++ [{ssl, true}, - {ssl_opts, SSLOpts}]; - undefined -> - WebConfig = WebConfig1 - end, - Web = {webmachine_mochiweb, - {webmachine_mochiweb, start, [WebConfig]}, - permanent, 5000, worker, dynamic}, - ServerSup = {stanchion_server_sup, - {stanchion_server_sup, start_link, []}, - permanent, 5000, worker, dynamic}, + {dispatch, stanchion_web:dispatch_table()}, + {ip, Ip}, + {port, Port}, + {nodelay, true}, + {log_dir, "log"}, + %% {rewrite_module, stanchion_wm_rewrite}, + {error_handler, stanchion_wm_error_handler} + ], + WebConfig = + case application:get_env(stanchion, ssl) of + {ok, SSLOpts} -> + WebConfig1 ++ [{ssl, true}, + {ssl_opts, SSLOpts}]; + undefined -> + WebConfig1 + end, + Web = #{id => webmachine_mochiweb, + start => {webmachine_mochiweb, start, [WebConfig]}, + restart => permanent, + shutdown => 5000, + modules => dynamic}, + ServerSup = #{id => stanchion_server_sup, + start => {stanchion_server_sup, start_link, []}, + restart => permanent, + shutdown => 5000, + modules => dynamic}, Processes = [ServerSup, Web], - {ok, { {one_for_one, 10, 10}, Processes} }. + {ok, { #{strategy => one_for_one, + intensity => 10, + period => 10}, Processes} }. diff --git a/src/stanchion_utils.erl b/apps/stanchion/src/stanchion_utils.erl similarity index 87% rename from src/stanchion_utils.erl rename to apps/stanchion/src/stanchion_utils.erl index 0a7f7b8..b5161d6 100644 --- a/src/stanchion_utils.erl +++ b/apps/stanchion/src/stanchion_utils.erl @@ -27,12 +27,14 @@ close_riak_connection/1, create_bucket/1, create_user/1, + get_user/2, + from_riakc_obj/2, delete_bucket/2, from_bucket_name/1, get_admin_creds/0, get_keys_and_values/1, - get_manifests/3, - get_manifests_raw/3, + get_manifests/4, + get_manifests_raw/4, has_tombstone/1, pow/2, pow/3, @@ -40,6 +42,7 @@ riak_connection/2, set_bucket_acl/2, set_bucket_policy/2, + set_bucket_versioning/2, delete_bucket_policy/2, timestamp/1, to_bucket_name/2, @@ -49,6 +52,8 @@ ]). -include("stanchion.hrl"). +-include_lib("rcs_common/include/rcs_common_manifest.hrl"). +-include_lib("rcs_common/include/rcs_common_moss.hrl"). -include_lib("riakc/include/riakc.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). @@ -57,15 +62,13 @@ -define(OBJECT_BUCKET_PREFIX, <<"0o:">>). % Version # = 0 -define(BLOCK_BUCKET_PREFIX, <<"0b:">>). % Version # = 0 --type bucket_op() :: create | update_acl | delete | update_policy | delete_policy. +-type bucket_op() :: create | update_acl | delete | update_policy | delete_policy | update_versioning. +-type bucket_op_option() :: {acl, acl()} + | {policy, binary()} + | delete_policy + | {bag, binary()} + | {versioning, binary()}. -type bucket_op_options() :: [bucket_op_option()]. --type bucket_op_option() :: {acl, acl()} | {policy, binary()} | delete_policy | {bag, binary()}. - --ifdef(namespaced_types). --type dictionary() :: dict:dict(). --else. --type dictionary() :: dict(). --endif. %% =================================================================== %% Public API @@ -178,10 +181,10 @@ get_admin_creds() -> end. %% @doc --spec get_manifests(pid(), binary(), binary()) -> +-spec get_manifests(pid(), binary(), binary(), binary()) -> {ok, term(), term()} | {error, notfound}. -get_manifests(RiakcPid, Bucket, Key) -> - case get_manifests_raw(RiakcPid, Bucket, Key) of +get_manifests(RiakcPid, Bucket, Key, Vsn) -> + case get_manifests_raw(RiakcPid, Bucket, Key, Vsn) of {ok, Object} -> DecodedSiblings = [binary_to_term(V) || {_, V}=Content <- riakc_obj:get_contents(Object), @@ -189,38 +192,40 @@ get_manifests(RiakcPid, Bucket, Key) -> %% Upgrade the manifests to be the latest erlang %% record version - Upgraded = stanchion_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), + Upgraded = rcs_common_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), %% resolve the siblings - Resolved = stanchion_manifest_resolution:resolve(Upgraded), + Resolved = rcs_common_manifest_resolution:resolve(Upgraded), %% prune old scheduled_delete manifests %% commented out because we don't have the %% riak_cs_gc module - %% Pruned = stanchion_manifest_utils:prune(Resolved), - {ok, Object, Resolved}; - {error, notfound}=NotFound -> + Pruned = rcs_common_manifest_utils:prune( + Resolved, erlang:timestamp(), + 50, %% riak_cs defaults for max_scheduled_delete_manifests and + 86400), %% leeway_seconds + {ok, Object, Pruned}; + {error, notfound} = NotFound -> NotFound end. %% @doc Determine if a set of contents of a riak object has a tombstone. --spec has_tombstone({dictionary(), binary()}) -> boolean(). +-spec has_tombstone({dict:dict(), binary()}) -> boolean(). has_tombstone({_, <<>>}) -> true; has_tombstone({MD, _V}) -> dict:is_key(<<"X-Riak-Deleted">>, MD) =:= true. %% @doc List the keys from a bucket --spec list_keys(binary(), pid()) -> {ok, [binary()]} | {error, term()}. list_keys(BucketName, RiakPid) -> case ?TURNAROUND_TIME(riakc_pb_socket:list_keys(RiakPid, BucketName)) of {{ok, Keys}, TAT} -> stanchion_stats:update([riakc, list_all_manifest_keys], TAT), {ok, lists:sort(Keys)}; - {{error, _}, TAT} = Else -> + {{error, _} = ER, TAT} -> stanchion_stats:update([riakc, list_all_manifest_keys], TAT), - Else + ER end. %% @doc Integer version of the standard pow() function; call the recursive accumulator to calculate. @@ -241,7 +246,7 @@ pow(Base, Power, Acc) -> %% @doc Store a new bucket in Riak %% though whole metadata itself is a dict, a metadata of ?MD_USERMETA is %% proplists of {?MD_ACL, ACL::binary()}|{?MD_POLICY, PolicyBin::binary()}| -%% {?MD_BAG, BagId::binary()}. +%% {?MD_BAG, BagId::binary()}, {?MD_VERSIONING, bucket_versioning_option()}}. %% should preserve other metadata. ACL and Policy can be overwritten. -spec put_bucket(term(), binary(), bucket_op_options(), pid()) -> ok | {error, term()}. @@ -263,7 +268,7 @@ put_bucket(BucketObj, OwnerId, Opts, RiakPid) -> stanchion_stats:update([riakc, put_cs_bucket], TAT), Result. --spec make_new_metadata(dictionary(), bucket_op_options()) -> dictionary(). +-spec make_new_metadata(dict:dict(), bucket_op_options()) -> dict:dict(). make_new_metadata(MD, Opts) -> MetaVals = dict:fetch(?MD_USERMETA, MD), UserMetaData = make_new_user_metadata(MetaVals, Opts), @@ -272,7 +277,7 @@ make_new_metadata(MD, Opts) -> -spec make_new_user_metadata(proplists:proplist(), bucket_op_options()) -> proplists:proplist(). make_new_user_metadata(MetaVals, [])-> MetaVals; -make_new_user_metadata(MetaVals, [{acl, Acl} | Opts])-> +make_new_user_metadata(MetaVals, [{acl, Acl} | Opts]) -> make_new_user_metadata(replace_meta(?MD_ACL, Acl, MetaVals), Opts); make_new_user_metadata(MetaVals, [{policy, Policy} | Opts]) -> make_new_user_metadata(replace_meta(?MD_POLICY, Policy, MetaVals), Opts); @@ -281,7 +286,9 @@ make_new_user_metadata(MetaVals, [{bag, undefined} | Opts]) -> make_new_user_metadata(MetaVals, [{bag, BagId} | Opts]) -> make_new_user_metadata(replace_meta(?MD_BAG, BagId, MetaVals), Opts); make_new_user_metadata(MetaVals, [delete_policy | Opts]) -> - make_new_user_metadata(proplists:delete(?MD_POLICY, MetaVals), Opts). + make_new_user_metadata(proplists:delete(?MD_POLICY, MetaVals), Opts); +make_new_user_metadata(MetaVals, [{versioning, VsnOpt} | Opts]) -> + make_new_user_metadata(replace_meta(?MD_VERSIONING, VsnOpt, MetaVals), Opts). replace_meta(Key, NewValue, MetaVals) -> [{Key, term_to_binary(NewValue)} | proplists:delete(Key, MetaVals)]. @@ -337,6 +344,13 @@ set_bucket_policy(Bucket, FieldList) -> % code of JSON parse from riak_cs_s3_policy do_bucket_op(Bucket, OwnerId, [{policy, PolicyJson}], update_policy). +%% @doc set bucket versioning option +-spec set_bucket_versioning(binary(), term()) -> ok | {error, term()}. +set_bucket_versioning(Bucket, FieldList) -> + OwnerId = proplists:get_value(<<"requester">>, FieldList, <<>>), + Json = proplists:get_value(<<"versioning">>, FieldList, []), + do_bucket_op(Bucket, OwnerId, [{versioning, Json}], update_versioning). + %% @doc Delete a bucket -spec delete_bucket_policy(binary(), binary()) -> ok | {error, term()}. @@ -380,7 +394,7 @@ update_user(KeyId, UserFields) -> {error, _}=Error -> Error end - catch T:E -> + catch T:E:_ST -> _ = lager:error("Error on updating user ~s: ~p", [KeyId, {T, E}]), {error, {T, E}} @@ -436,7 +450,8 @@ bucket_empty_any_pred(RiakcPid, Bucket) -> %% we'll take care of calling `to_bucket_name' -spec key_exists(pid(), binary(), binary()) -> boolean(). key_exists(RiakcPid, Bucket, Key) -> - key_exists_handle_get_manifests(get_manifests(RiakcPid, Bucket, Key)). + key_exists_handle_get_manifests( + get_manifests(RiakcPid, Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION)). %% @private -spec key_exists_handle_get_manifests({ok, riakc_obj:riakc_obj(), list()} | @@ -458,7 +473,7 @@ active_to_bool({error, notfound}) -> {error, notfound}) -> {ok, term()} | {error, notfound}. active_manifest_from_response({ok, Manifests}) -> - handle_active_manifests(stanchion_manifest_utils:active_manifest(Manifests)); + handle_active_manifests(rcs_common_manifest_utils:active_manifest(Manifests)); active_manifest_from_response({error, notfound}=NotFound) -> NotFound. @@ -504,13 +519,16 @@ bucket_available(Bucket, RequesterId, BucketOp, RiakPid) -> {false, no_such_bucket}; update_policy -> {false, no_such_bucket}; + update_versioning -> + {false, no_such_bucket}; delete -> {false, no_such_bucket} end; {{error, Reason}, TAT} -> stanchion_stats:update([riakc, get_cs_bucket], TAT), %% @TODO Maybe bubble up this error info - _ = lager:warning("Error occurred trying to check if the bucket ~p exists. Reason: ~p", [Bucket, Reason]), + _ = lager:warning("Error occurred trying to check if the bucket ~p exists. Reason: ~p", + [Bucket, Reason]), {false, Reason} end. @@ -528,11 +546,12 @@ do_bucket_op(Bucket, OwnerId, Opts, BucketOp) -> case bucket_available(Bucket, OwnerId, BucketOp, RiakPid) of {true, BucketObj} -> Value = case BucketOp of - create -> OwnerId; - update_acl -> OwnerId; - update_policy -> OwnerId; - delete_policy -> OwnerId; - delete -> ?FREE_BUCKET_MARKER + create -> OwnerId; + update_acl -> OwnerId; + update_policy -> OwnerId; + delete_policy -> OwnerId; + update_versioning -> OwnerId; + delete -> ?FREE_BUCKET_MARKER end, put_bucket(BucketObj, Value, Opts, RiakPid); {false, Reason1} -> @@ -596,15 +615,15 @@ is_bucket_clean(Bucket, RiakPid, BucketObj) -> {true, BucketObj} end end - catch T:E -> + catch T:E:ST -> _ = lager:error("Could not check whether bucket was empty. Reason: ~p:~p - ~p", - [T,E,erlang:get_stacktrace()]), + [T, E, ST]), error({T, E}) after close_manifest_connection(RiakPid, ManifestRiakPid) end. --spec manifest_connection(pid, riakc_obj:riakc_obj()) -> {ok, pid()} | {error, term()}. +-spec manifest_connection(pid(), riakc_obj:riakc_obj()) -> {ok, pid()} | {error, term()}. manifest_connection(RiakPid, BucketObj) -> case bag_id_from_bucket(BucketObj) of undefined -> {ok, RiakPid}; @@ -616,8 +635,6 @@ manifest_connection(RiakPid, BucketObj) -> end end. --spec conn_info_from_bag(binary(), undefined | {ok, [{string(), string(), non_neg_integer()}]}) - -> {string(), non_neg_integer()}. conn_info_from_bag(_BagId, undefined) -> undefined; conn_info_from_bag(BagId, {ok, Bags}) -> @@ -629,7 +646,6 @@ conn_info_from_bag(BagId, {ok, Bags}) -> {Address, Port} end. --spec bag_id_from_bucket(riakc_obj:riakc_obj()) -> binary(). bag_id_from_bucket(BucketObj) -> Contents = riakc_obj:get_contents(BucketObj), bag_id_from_contents(Contents). @@ -663,8 +679,15 @@ close_manifest_connection(_RiakPid, ManifestRiakPid) -> %% for a particular key. %% @TODO Consider other options that would give more %% assurance that a particular email address is available. --spec email_available(binary(), pid()) -> true | {false, term()}. -email_available(Email, RiakPid) -> +-spec email_available(string() | binary(), pid()) -> true | {false, user_already_exists | term()}. +email_available(Email_, RiakPid) -> + + %% this is to pacify dialyzer, which makes an issue of + %% Email::string() (coming from #rcs_user_v2.email type) as + %% conflicting with binary() as the type appropriate for 4th arg + %% in get_index_eq + Email = iolist_to_binary([Email_]), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:get_index_eq(RiakPid, ?USER_BUCKET, ?EMAIL_INDEX, Email)), stanchion_stats:update([riakc, get_user_by_index], TAT), case Res of @@ -674,7 +697,8 @@ email_available(Email, RiakPid) -> {false, user_already_exists}; {error, Reason} -> %% @TODO Maybe bubble up this error info - _ = lager:warning("Error occurred trying to check if the address ~p has been registered. Reason: ~p", [Email, Reason]), + lager:warning("Error occurred trying to check if the address ~p has been registered. Reason: ~p", + [Email, Reason]), {false, Reason} end. @@ -684,15 +708,16 @@ email_available(Email, RiakPid) -> get_keys_and_values(BucketName) -> case riak_connection() of {ok, RiakPid} -> - case list_keys(BucketName, RiakPid) of - {ok, Keys} -> - KeyValuePairs = - [{Key, get_value(BucketName, Key, RiakPid)} - || Key <- Keys], - Res = {ok, KeyValuePairs}; - {error, Reason1} -> - Res = {error, Reason1} - end, + Res = + case list_keys(BucketName, RiakPid) of + {ok, Keys} -> + KeyValuePairs = + [{Key, get_value(BucketName, Key, RiakPid)} + || Key <- Keys], + {ok, KeyValuePairs}; + {error, Reason1} -> + {error, Reason1} + end, close_riak_connection(RiakPid), Res; {error, _} = Else -> @@ -701,11 +726,13 @@ get_keys_and_values(BucketName) -> %% internal fun to retrieve the riak object %% at a bucket/key --spec get_manifests_raw(pid(), binary(), binary()) -> +-spec get_manifests_raw(pid(), binary(), binary(), binary()) -> {ok, riakc_obj:riakc_obj()} | {error, notfound}. -get_manifests_raw(RiakcPid, Bucket, Key) -> +get_manifests_raw(RiakcPid, Bucket, Key, Vsn) -> ManifestBucket = to_bucket_name(objects, Bucket), - {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:get(RiakcPid, ManifestBucket, Key)), + {Res, TAT} = ?TURNAROUND_TIME( + riakc_pb_socket:get(RiakcPid, ManifestBucket, + rcs_common_manifest:make_versioned_key(Key, Vsn))), stanchion_stats:update([riakc, get_manifest], TAT), Res. @@ -779,25 +806,28 @@ get_user(KeyId, RiakPid) -> BinKey = list_to_binary(KeyId), case fetch_user(BinKey, RiakPid) of {ok, {Obj, KeepDeletedBuckets}} -> - case riakc_obj:value_count(Obj) of - 1 -> - User = binary_to_term(riakc_obj:get_value(Obj)), - {ok, {User, Obj}}; - 0 -> - {error, no_value}; - _ -> - Values = [binary_to_term(Value) || - Value <- riakc_obj:get_values(Obj), - Value /= <<>> % tombstone - ], - User = hd(Values), - Buckets = resolve_buckets(Values, [], KeepDeletedBuckets), - {ok, {User?RCS_USER{buckets=Buckets}, Obj}} - end; + from_riakc_obj(Obj, KeepDeletedBuckets); Error -> Error end. +from_riakc_obj(Obj, KeepDeletedBuckets) -> + case riakc_obj:value_count(Obj) of + 1 -> + User = binary_to_term(riakc_obj:get_value(Obj)), + {ok, {User, Obj}}; + 0 -> + {error, no_value}; + _ -> + Values = [binary_to_term(Value) || + Value <- riakc_obj:get_values(Obj), + Value /= <<>> % tombstone + ], + User = hd(Values), + Buckets = resolve_buckets(Values, [], KeepDeletedBuckets), + {ok, {User?RCS_USER{buckets=Buckets}, Obj}} + end. + %% @doc Perform an initial read attempt with R=PR=N. %% If the initial read fails retry using %% R=quorum and PR=1, but indicate that bucket deletion diff --git a/src/stanchion_web.erl b/apps/stanchion/src/stanchion_web.erl similarity index 91% rename from src/stanchion_web.erl rename to apps/stanchion/src/stanchion_web.erl index ae1b64d..cb2e5da 100644 --- a/src/stanchion_web.erl +++ b/apps/stanchion/src/stanchion_web.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -34,6 +35,7 @@ dispatch_table() -> {["buckets"], stanchion_wm_buckets, [{auth_bypass, AuthBypass}]}, {["buckets", bucket, "acl"], stanchion_wm_acl, [{auth_bypass, AuthBypass}]}, {["buckets", bucket, "policy"], stanchion_wm_policy, [{auth_bypass, AuthBypass}]}, + {["buckets", bucket, "versioning"], stanchion_wm_versioning, [{auth_bypass, AuthBypass}]}, {["buckets", bucket], stanchion_wm_bucket, [{auth_bypass, AuthBypass}]}, {["users", key_id], stanchion_wm_user, [{auth_bypass, AuthBypass}]}, {["users"], stanchion_wm_users, [{auth_bypass, AuthBypass}]}, diff --git a/src/stanchion_wm_acl.erl b/apps/stanchion/src/stanchion_wm_acl.erl similarity index 98% rename from src/stanchion_wm_acl.erl rename to apps/stanchion/src/stanchion_wm_acl.erl index a24555f..e7dd2eb 100644 --- a/src/stanchion_wm_acl.erl +++ b/apps/stanchion/src/stanchion_wm_acl.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_wm_bucket.erl b/apps/stanchion/src/stanchion_wm_bucket.erl similarity index 96% rename from src/stanchion_wm_bucket.erl rename to apps/stanchion/src/stanchion_wm_bucket.erl index fee8354..275e986 100644 --- a/src/stanchion_wm_bucket.erl +++ b/apps/stanchion/src/stanchion_wm_bucket.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -91,9 +92,7 @@ content_types_accepted(RD, Ctx) -> {{'halt', _}, #wm_reqdata{}, term()}. to_xml(RD, Ctx) -> Bucket = wrq:path_info(bucket, RD), - stanchion_response:list_buckets_response(Bucket, - RD, - Ctx). + stanchion_response:list_buckets_response(Bucket, RD, Ctx). %% SLF: Dialyzer note: As far as I can tell, the func %% stanchion_utils:update_bucket_owner() has never existed. diff --git a/src/stanchion_wm_buckets.erl b/apps/stanchion/src/stanchion_wm_buckets.erl similarity index 98% rename from src/stanchion_wm_buckets.erl rename to apps/stanchion/src/stanchion_wm_buckets.erl index 8c4a960..2831e76 100644 --- a/src/stanchion_wm_buckets.erl +++ b/apps/stanchion/src/stanchion_wm_buckets.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_wm_error_handler.erl b/apps/stanchion/src/stanchion_wm_error_handler.erl similarity index 92% rename from src/stanchion_wm_error_handler.erl rename to apps/stanchion/src/stanchion_wm_error_handler.erl index 624e8cc..e863461 100644 --- a/src/stanchion_wm_error_handler.erl +++ b/apps/stanchion/src/stanchion_wm_error_handler.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,7 +25,7 @@ render_error(500, Req, Reason) -> {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), {Path,_} = Req:path(), - error_logger:error_msg("webmachine error: path=~p~n~p~n", [Path, Reason]), + lager:error("webmachine error: path=~p~n~p~n", [Path, Reason]), STString = io_lib:format("~p", [Reason]), ErrorStart = "500 Internal Server Error

Internal Server Error

The server encountered an error while processing this request:
",
     ErrorEnd = "

mochiweb+webmachine web server
", diff --git a/src/stanchion_wm_policy.erl b/apps/stanchion/src/stanchion_wm_policy.erl similarity index 95% rename from src/stanchion_wm_policy.erl rename to apps/stanchion/src/stanchion_wm_policy.erl index 34c753d..37a3836 100644 --- a/src/stanchion_wm_policy.erl +++ b/apps/stanchion/src/stanchion_wm_policy.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -101,7 +102,7 @@ accept_body(RD, Ctx) -> end. %% @doc Callback for deleting an object. --spec delete_resource(#wm_reqdata{}, #context{}) -> {true, #wm_reqdata{}, #context{}}. +-spec delete_resource(#wm_reqdata{}, #context{}) -> {{halt, 204}, #wm_reqdata{}, #context{}} | {true, #wm_reqdata{}, #context{}}. delete_resource(RD, Ctx) -> Bucket = list_to_binary(wrq:path_info(bucket, RD)), RequesterId = list_to_binary(wrq:get_qs_value("requester", "", RD)), @@ -114,4 +115,3 @@ delete_resource(RD, Ctx) -> {error, Reason} -> stanchion_response:api_error(Reason, RD, Ctx) end. - diff --git a/src/stanchion_wm_stats.erl b/apps/stanchion/src/stanchion_wm_stats.erl similarity index 97% rename from src/stanchion_wm_stats.erl rename to apps/stanchion/src/stanchion_wm_stats.erl index 277f976..29b9fe9 100644 --- a/src/stanchion_wm_stats.erl +++ b/apps/stanchion/src/stanchion_wm_stats.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_wm_user.erl b/apps/stanchion/src/stanchion_wm_user.erl similarity index 98% rename from src/stanchion_wm_user.erl rename to apps/stanchion/src/stanchion_wm_user.erl index b292942..68360e0 100644 --- a/src/stanchion_wm_user.erl +++ b/apps/stanchion/src/stanchion_wm_user.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_wm_users.erl b/apps/stanchion/src/stanchion_wm_users.erl similarity index 98% rename from src/stanchion_wm_users.erl rename to apps/stanchion/src/stanchion_wm_users.erl index ee79ba5..b6598ec 100644 --- a/src/stanchion_wm_users.erl +++ b/apps/stanchion/src/stanchion_wm_users.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/stanchion_wm_utils.erl b/apps/stanchion/src/stanchion_wm_utils.erl similarity index 98% rename from src/stanchion_wm_utils.erl rename to apps/stanchion/src/stanchion_wm_utils.erl index e5a02a8..4685799 100644 --- a/src/stanchion_wm_utils.erl +++ b/apps/stanchion/src/stanchion_wm_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/apps/stanchion/src/stanchion_wm_versioning.erl b/apps/stanchion/src/stanchion_wm_versioning.erl new file mode 100644 index 0000000..ebb589b --- /dev/null +++ b/apps/stanchion/src/stanchion_wm_versioning.erl @@ -0,0 +1,86 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_versioning). + +-export([init/1, + service_available/2, + is_authorized/2, + malformed_request/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + % eval every atoms to make binary_to_existing_atom/2 work. + {ok, #context{auth_bypass=AuthBypass}}. + +-spec service_available(term(), term()) -> {true, term(), term()}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec malformed_request(term(), term()) -> {false, term(), term()}. +malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + +%% @doc Check that the request is from the admin user +is_authorized(RD, Ctx=#context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + {true, RD, Ctx}; + {error, _Reason} -> + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['PUT'], RD, Ctx}. + +-spec content_types_accepted(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +%% @doc Process the request body on `PUT'. +accept_body(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + Body = wrq:req_body(RD), + %% @TODO Handle json decoding exceptions + ParsedBody = mochijson2:decode(Body), + FieldList = stanchion_wm_utils:json_to_proplist(ParsedBody), + case stanchion_server:set_bucket_versioning(Bucket, + FieldList) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/stanchion/test/stanchion_config_test.erl b/apps/stanchion/test/stanchion_config_test.erl new file mode 100644 index 0000000..7ae3693 --- /dev/null +++ b/apps/stanchion/test/stanchion_config_test.erl @@ -0,0 +1,31 @@ +-module(stanchion_config_test). +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +default_config_test() -> + Config = cuttlefish_unit:generate_templated_config(schema_files(), [], context()), + cuttlefish_unit:assert_config(Config, "stanchion.host", {"127.0.0.1", 8085}), + cuttlefish_unit:assert_config(Config, "stanchion.riak_host", {"127.0.0.1", 8087}), + cuttlefish_unit:assert_not_configured(Config, "stanchion.ssl"), + cuttlefish_unit:assert_config(Config, "stanchion.admin_key", "admin-key"), + cuttlefish_unit:assert_not_configured(Config, "stanchion.admin_secret"), + cuttlefish_unit:assert_config(Config, "stanchion.auth_bypass", false), + + ok. + +ssl_config_test() -> + Conf = [{["ssl", "certfile"], "path/certfile"}, + {["ssl", "keyfile"], "path/keyfile"}], + Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), + cuttlefish_unit:assert_config(Config, "stanchion.ssl", [{keyfile, "path/keyfile"}, + {certfile, "path/certfile"}]), + ok. + +schema_files() -> + ["priv/stanchion.schema"]. + +context() -> + {ok, Context} = file:consult("rel/vars.config"), + Context. diff --git a/config/sys.config.defaults b/config/sys.config.defaults new file mode 100644 index 0000000..67812a3 --- /dev/null +++ b/config/sys.config.defaults @@ -0,0 +1,10 @@ +PLATFORM_LOG_DIR="./log" +PLATFORM_LIB_DIR="./lib" +PLATFORM_ETC_DIR="./etc" +PLATFORM_BIN_DIR="./bin" +AUTH_BYPASS=false +ADMIN_KEY="admin-key" +RIAK_HOST="127.0.0.1" +RIAK_PORT=8087 +LISTENER_HOST="127.0.0.1" +LISTENER_PORT=8085 diff --git a/config/sys.docker.config.src b/config/sys.docker.config.src new file mode 100644 index 0000000..13976c7 --- /dev/null +++ b/config/sys.docker.config.src @@ -0,0 +1,31 @@ +[{lager, + [{error_logger_hwm,100}, + {error_logger_redirect,true}, + {crash_log_date,"$D0"}, + {crash_log_size,10485760}, + {crash_log_msg_size,65536}, + {handlers, + [{lager_file_backend, + [{file, ${PLATFORM_LOG_DIR}"/console.log"}, + {level, info}, + {size, 10485760}, + {date, "$D0"}, + {count, 5}]}, + {lager_file_backend, + [{file, ${PLATFORM_LOG_DIR}"/error.log"}, + {level, error}, + {size, 10485760}, + {date, "$D0"}, + {count, 5}]}]}, + {crash_log, ${PLATFORM_LOG_DIR}"/crash.log"}, + {crash_log_count, 5}]}, + {sasl,[{sasl_error_logger, false}]}, + {stanchion, + [{platform_log_dir, ${PLATFORM_LOG_DIR}}, + {platform_lib_dir, ${PLATFORM_LIB_DIR}}, + {platform_etc_dir, ${PLATFORM_ETC_DIR}}, + {platform_bin_dir, ${PLATFORM_BIN_DIR}}, + {auth_bypass, ${AUTH_BYPASS}}, + {admin_key, ${ADMIN_KEY}}, + {riak_host, {${RIAK_HOST}, ${RIAK_PORT}}}, + {host, {${LISTENER_HOST}, ${LISTENER_PORT}}}]}]. diff --git a/config/vm.args b/config/vm.args new file mode 100644 index 0000000..7f08b8b --- /dev/null +++ b/config/vm.args @@ -0,0 +1,2 @@ +-setcookie riak +-name stanchion@127.0.0.1 diff --git a/include/riak_moss.hrl b/include/riak_moss.hrl deleted file mode 100644 index 70ded17..0000000 --- a/include/riak_moss.hrl +++ /dev/null @@ -1,338 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --record(moss_user, { - name :: string(), - key_id :: string(), - key_secret :: string(), - buckets = []}). - --record(moss_user_v1, { - name :: string(), - display_name :: string(), - email :: string(), - key_id :: string(), - key_secret :: string(), - canonical_id :: string(), - buckets=[] :: [moss_bucket()]}). - --record(rcs_user_v2, { - name :: string(), - display_name :: string(), - email :: string(), - key_id :: string(), - key_secret :: string(), - canonical_id :: string(), - buckets=[] :: [moss_bucket()], - status=enabled :: enabled | disabled}). --type moss_user() :: #rcs_user_v2{} | #moss_user_v1{}. --type rcs_user() :: #rcs_user_v2{} | #moss_user_v1{}. - --record(moss_bucket, { - name :: string(), - creation_date :: term(), - acl :: acl()}). - --record(moss_bucket_v1, { - name :: string(), - last_action :: created | deleted, - creation_date :: string(), - modification_time :: erlang:timestamp(), - acl :: acl()}). --type moss_bucket() :: #moss_bucket_v1{}. --type bucket_operation() :: create | delete | update_acl. --type bucket_action() :: created | deleted. - --record(context, {start_time :: erlang:timestamp(), - auth_bypass :: atom(), - user :: moss_user(), - user_vclock :: term(), - bucket :: binary(), - requested_perm :: acl_perm(), - riakc_pid :: pid() - }). - --record(key_context, {context :: #context{}, - manifest :: 'notfound' | lfs_manifest(), - get_fsm_pid :: pid(), - putctype :: string(), - bucket :: binary(), - key :: list(), - owner :: 'undefined' | string(), - size :: non_neg_integer()}). - --type acl_perm() :: 'READ' | 'WRITE' | 'READ_ACP' | 'WRITE_ACP' | 'FULL_CONTROL'. --type acl_perms() :: [acl_perm()]. --type group_grant() :: 'AllUsers' | 'AuthUsers'. --type acl_grantee() :: {string(), string()} | group_grant(). --type acl_grant() :: {acl_grantee(), acl_perms()}. --type acl_owner() :: {string(), string()} | {string(), string(), string()}. --record(acl_v1, {owner={"", ""} :: acl_owner(), - grants=[] :: [acl_grant()], - creation_time=now() :: erlang:timestamp()}). --record(acl_v2, {owner={"", "", ""} :: acl_owner(), - grants=[] :: [acl_grant()], - creation_time=now() :: erlang:timestamp()}). --type acl() :: #acl_v1{} | #acl_v2{}. - --type cluster_id() :: undefined | term(). % Type still in flux. - --type cs_uuid() :: binary(). - --record(lfs_manifest_v2, { - version=2 :: integer(), - block_size :: integer(), - bkey :: {binary(), binary()}, - metadata :: orddict:orddict(), - created=riak_moss_wm_utils:iso_8601_datetime(), - uuid :: cs_uuid(), - content_length :: non_neg_integer(), - content_type :: binary(), - content_md5 :: term(), - state=undefined :: undefined | writing | active | - pending_delete | scheduled_delete | deleted, - write_start_time :: term(), %% immutable - last_block_written_time :: term(), - write_blocks_remaining :: ordsets:ordset(integer()), - delete_marked_time :: term(), - last_block_deleted_time :: term(), - delete_blocks_remaining :: ordsets:ordset(integer()), - acl :: acl(), - props = [] :: proplists:proplist(), - cluster_id :: cluster_id() - }). - --record(lfs_manifest_v3, { - %% "global" properties - %% ----------------------------------------------------------------- - - %% this isn't as important anymore - %% since we're naming the record - %% to include the version number, - %% but I figured it's worth keeping - %% in case we change serialization - %% formats in the future. - version=3 :: integer(), - - %% the block_size setting when this manifest - %% was written. Needed if the user - %% ever changes the block size after writing - %% data - block_size :: integer(), - - %% identifying properties - %% ----------------------------------------------------------------- - bkey :: {binary(), binary()}, - - %% user metadata that would normally - %% be placed on the riak_object. We avoid - %% putting it on the riak_object so that - %% we can use that metadata ourselves - metadata :: orddict:orddict(), - - %% the date the manifest was created. - %% not sure if we need both this and - %% write_start_time. My thought was that - %% write_start_time would have millisecond - %% resolution, but I suppose there's no - %% reason we can't change created - %% to have millisecond as well. - created=riak_moss_wm_utils:iso_8601_datetime(), - uuid :: cs_uuid(), - - %% content properties - %% ----------------------------------------------------------------- - content_length :: non_neg_integer(), - content_type :: binary(), - content_md5 :: term(), - - %% state properties - %% ----------------------------------------------------------------- - state=undefined :: undefined | writing | active | - pending_delete | scheduled_delete | deleted, - - %% writing/active state - %% ----------------------------------------------------------------- - write_start_time :: term(), %% immutable - - %% used for two purposes - %% 1. to mark when a file has finished uploading - %% 2. to decide if a write crashed before completing - %% and needs to be garbage collected - last_block_written_time :: term(), - - %% a shrink-only (during resolution) - %% set to denote which blocks still - %% need to be written. We use a shrinking - %% (rather than growing) set to that the - %% set is empty after the write has completed, - %% which should be most of the lifespan on disk - write_blocks_remaining :: ordsets:ordset(integer()), - - %% pending_delete/deleted state - %% ----------------------------------------------------------------- - %% set to the current time - %% when a manifest is marked as deleted - %% and enters the pending_delete state - delete_marked_time :: term(), - - %% the timestamp serves a similar - %% purpose to last_block_written_time, - %% in that it's used for figuring out - %% when delete processes have died - %% and garbage collection needs to - %% pick up where they left off. - last_block_deleted_time :: term(), - - %% a shrink-only (during resolution) - %% set to denote which blocks - %% still need to be deleted. - %% See write_blocks_remaining for - %% an explanation of why we chose - %% a shrinking set - delete_blocks_remaining :: ordsets:ordset(integer()), - - %% the time the manifest was put - %% into the scheduled_delete - %% state - scheduled_delete_time :: term(), - - %% The ACL for the version of the object represented - %% by this manifest. - acl :: acl(), - - %% There are a couple of cases where we want to add record - %% member'ish data without adding new members to the record, - %% e.g. - %% 1. Data for which the common value is 'undefined' or not - %% used/set for this particular manifest - %% 2. Cases where we do want to change the structure of the - %% record but don't want to go through the full code - %% refactoring and backward-compatibility tap dance - %% until sometime later. - props = [] :: proplists:proplist(), - - %% cluster_id: A couple of uses, both short- and longer-term - %% possibilities: - %% - %% 1. We don't have a good story in early 2012 for how to - %% build a stable 2,000 node Riak cluster. If MOSS can - %% talk to multiple Riak clusters, then each individual - %% cluster can be a size that we're comfortable - %% supporting. - %% - %% 2. We may soon have Riak EE's replication have full - %% plumbing to make it feasible to forward arbitrary - %% traffic between clusters. Then if a slave cluster is - %% missing a data block, and read-repair cannot - %% automagically fix the 'not_found' problem, then perhaps - %% forwarding a get request to the source Riak cluster can - %% fetch us the missing data. - cluster_id :: cluster_id() - }). --type lfs_manifest() :: #lfs_manifest_v3{}. - --type cs_uuid_and_manifest() :: {cs_uuid(), lfs_manifest()}. - --define(MANIFEST, #lfs_manifest_v3). - --define(ACL, #acl_v2). --define(MOSS_BUCKET, #moss_bucket_v1). --define(MOSS_USER, #rcs_user_v2). --define(RCS_USER, #rcs_user_v2). --define(USER_BUCKET, <<"moss.users">>). --define(ACCESS_BUCKET, <<"moss.access">>). --define(STORAGE_BUCKET, <<"moss.storage">>). --define(BUCKETS_BUCKET, <<"moss.buckets">>). --define(GC_BUCKET, <<"riak-cs-gc">>). --define(FREE_BUCKET_MARKER, <<"0">>). --define(DEFAULT_MAX_CONTENT_LENGTH, 5368709120). %% 5 GB --define(DEFAULT_LFS_BLOCK_SIZE, 1048576).%% 1 MB --define(XML_PROLOG, ""). --define(DEFAULT_STANCHION_IP, "127.0.0.1"). --define(DEFAULT_STANCHION_PORT, 8085). --define(DEFAULT_STANCHION_SSL, true). --define(MD_ACL, <<"X-Moss-Acl">>). --define(EMAIL_INDEX, <<"email_bin">>). --define(ID_INDEX, <<"c_id_bin">>). --define(KEY_INDEX, <<"$key">>). --define(AUTH_USERS_GROUP, "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"). --define(ALL_USERS_GROUP, "http://acs.amazonaws.com/groups/global/AllUsers"). --define(LOG_DELIVERY_GROUP, "http://acs.amazonaws.com/groups/s3/LogDelivery"). --define(DEFAULT_FETCH_CONCURRENCY, 1). --define(DEFAULT_PUT_CONCURRENCY, 1). --define(DEFAULT_DELETE_CONCURRENCY, 1). -%% A number to multiplied with the block size -%% to determine the PUT buffer size. -%% ex. 2 would mean BlockSize * 2 --define(DEFAULT_PUT_BUFFER_FACTOR, 1). --define(DEFAULT_PING_TIMEOUT, 5000). --define(JSON_TYPE, "application/json"). --define(XML_TYPE, "application/xml"). - -%% Major categories of Erlang-triggered DTrace probes -%% -%% The main R15B01 USDT probe that can be triggered by Erlang code is defined -%% like this: -%% -%% /** -%% * Multi-purpose probe: up to 4 NUL-terminated strings and 4 -%% * 64-bit integer arguments. -%% * -%% * @param proc, the PID (string form) of the sending process -%% * @param user_tag, the user tag of the sender -%% * @param i1, integer -%% * @param i2, integer -%% * @param i3, integer -%% * @param i4, integer -%% * @param s1, string/iolist. D's arg6 is NULL if not given by Erlang -%% * @param s2, string/iolist. D's arg7 is NULL if not given by Erlang -%% * @param s3, string/iolist. D's arg8 is NULL if not given by Erlang -%% * @param s4, string/iolist. D's arg9 is NULL if not given by Erlang -%% */ -%% probe user_trace__i4s4(char *proc, char *user_tag, -%% int i1, int i2, int i3, int i4, -%% char *s1, char *s2, char *s3, char *s4); -%% -%% The convention that we'll use of these probes is: -%% param D arg name use -%% ----- ---------- --- -%% i1 arg2 Application category (see below) -%% i2 arg3 1 = function entry, 2 = function return -%% NOTE! Not all function entry probes have a return probe -%% i3-i4 arg4-arg5 Varies, zero usually means unused (but not always!) -%% s1 arg6 Module name -%% s2 arg7 Function name -%% s3-4 arg8-arg9 Varies, NULL means unused -%% --define(DT_BLOCK_OP, 700). --define(DT_SERVICE_OP, 701). --define(DT_BUCKET_OP, 702). --define(DT_OBJECT_OP, 703). -%% perhaps add later? -define(DT_AUTH_OP, 704). --define(DT_WM_OP, 705). - -%% Number of seconds to keep manifests in the `scheduled_delete' state -%% before beginning to delete the file blocks and before the file -%% manifest may be pruned. --define(DEFAULT_LEEWAY_SECONDS, 86400). %% 24-hours --define(DEFAULT_GC_INTERVAL, 900). %% 15 minutes --define(DEFAULT_GC_RETRY_INTERVAL, 21600). %% 6 hours --define(EPOCH_START, <<"0">>). diff --git a/include/stanchion.hrl b/include/stanchion.hrl deleted file mode 100644 index b7553b9..0000000 --- a/include/stanchion.hrl +++ /dev/null @@ -1,100 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --record(context, {auth_bypass :: boolean(), - bucket :: binary(), - owner_id :: all | string()}). - --record(moss_user_v1, { - name :: string(), - display_name :: string(), - email :: string(), - key_id :: string(), - key_secret :: string(), - canonical_id :: string(), - buckets=[] :: [moss_bucket()]}). - --record(rcs_user_v2, { - name :: string(), - display_name :: string(), - email :: string(), - key_id :: string(), - key_secret :: string(), - canonical_id :: string(), - buckets=[] :: [moss_bucket()], - status=enabled :: enabled | disabled}). --type moss_user() :: #rcs_user_v2{} | #moss_user_v1{}. --type rcs_user() :: #rcs_user_v2{} | #moss_user_v1{}. - --record(moss_bucket_v1, { - name :: string(), - last_action :: created | deleted, - creation_date :: string(), - modification_time :: erlang:timestamp(), - acl :: acl()}). - --type moss_bucket() :: #moss_bucket_v1{}. --type cs_bucket() :: #moss_bucket_v1{}. - --type acl_perm() :: 'READ' | 'WRITE' | 'READ_ACP' | 'WRITE_ACP' | 'FULL_CONTROL'. --type acl_perms() :: [acl_perm()]. --type acl_grant() :: {{string(), string()}, acl_perms()} | {atom(), acl_perms()}. --type acl_owner() :: {string(), string()} | {string(), string(), string()}. --record(acl_v1, {owner={"", ""} :: acl_owner(), - grants=[] :: [acl_grant()], - creation_time=now() :: erlang:timestamp()}). --record(acl_v2, {owner={"", "", ""} :: acl_owner(), - grants=[] :: [acl_grant()], - creation_time=now() :: erlang:timestamp()}). --type acl1() :: #acl_v1{}. --type acl2() :: #acl_v2{}. --type acl() :: acl1() | acl2(). - --record(access_v1, { - method :: atom(), % PUT / GET / POST / .... - target :: atom(), % object | object_acl | .... - id :: binary(), - bucket :: binary(), - key = <<>> :: binary(), - req %:: #wm_reqdata{} % request of webmachine - }). - --type access() :: #access_v1{}. - --define(ACL, #acl_v2). --define(RCS_BUCKET, #moss_bucket_v1). --define(USER_BUCKET, <<"moss.users">>). --define(BUCKETS_BUCKET, <<"moss.buckets">>). --define(FREE_BUCKET_MARKER, <<"0">>). --define(MOSS_USER, #rcs_user_v2). --define(RCS_USER, #rcs_user_v2). --define(MD_ACL, <<"X-Moss-Acl">>). --define(MD_POLICY, <<"X-Rcs-Policy">>). --define(MD_BAG, <<"X-Rcs-Bag">>). - --define(TURNAROUND_TIME(Call), - begin - StartTime_____tat = os:timestamp(), - Result_____tat = (Call), - EndTime_____tat = os:timestamp(), - {Result_____tat, - timer:now_diff(EndTime_____tat, - StartTime_____tat)} - end). diff --git a/pkg.vars.config b/pkg.vars.config deleted file mode 100644 index 8783dd4..0000000 --- a/pkg.vars.config +++ /dev/null @@ -1,37 +0,0 @@ -%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et - -%% -%% Packaging -%% -{package_name, "stanchion"}. -{package_install_name, "stanchion"}. -{package_install_user, "stanchion"}. -{package_install_group, "riak"}. -{package_install_user_desc, "Stanchion user"}. -{package_shortdesc, "Request Serialization Application"}. -{package_desc, "Request Serialization Application"}. -{package_commands, {list, [[{name, "stanchion"}]]}}. -{package_patch_dir, "basho-patches"}. -{bin_or_sbin, "sbin"}. -{license_type, "Apache 2"}. -{copyright, "2013 Basho Technologies, Inc"}. -{vendor_name, "Basho Technologies, Inc"}. -{vendor_url, "http://basho.com"}. -{vendor_contact_name, "Basho Package Maintainer"}. -{vendor_contact_email, "packaging@basho.com"}. -{license_full_text, - "This file is provided to you under the Apache License,\n" - "Version 2.0 (the \"License\"); you may not use this file\n" - "except in compliance with the License. You may obtain\n" - "a copy of the License at\n" - "\n" - " http://www.apache.org/licenses/LICENSE-2.0\n" - "\n" - "Unless required by applicable law or agreed to in writing,\n" - "software distributed under the License is distributed on an\n" - "\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" - "KIND, either express or implied. See the License for the\n" - "specific language governing permissions and limitations\n" - "under the License."}. -{solaris_pkgname, "BASHOstanchion"}. diff --git a/rel/files/stanchion.schema b/priv/stanchion.schema similarity index 87% rename from rel/files/stanchion.schema rename to priv/stanchion.schema index 4248ca1..f5eb427 100644 --- a/rel/files/stanchion.schema +++ b/priv/stanchion.schema @@ -1,5 +1,15 @@ %%-*- mode: erlang -*- +%% @doc Version number. Intended use is to facilitate selective +%% support of features in the context of migration, where nodes +%% running different versions coexist in the same cluster. As +%% stanchion is not intended to ever run in clusters, this setting is +%% only used for reporting (via `stanchion admin version`). +{mapping, "stanchion_version", "stanchion.stanchion_version", [ + {default, {{stanchion_version}} }, + {datatype, integer} +]}. + %% @doc listen port and IP address {mapping, "listener", "stanchion.host", [ {default, {"{{stanchion_ip}}" , {{stanchion_port}} }}, @@ -29,15 +39,17 @@ ]}. %% @doc Admin user credentials. The credentials specified here must -%% match the admin credentials specified in the riak-cs app.config -%% for the system to function properly. +%% match the admin credentials specified in the riak-cs.conf for the +%% system to function properly. {mapping, "admin.key", "stanchion.admin_key", [ {default, "{{admin_key}}"}, {datatype, string} ]}. + +%% @doc admin.secret is deprecated and will be removed at next version. {mapping, "admin.secret", "stanchion.admin_secret", [ - {default, "{{admin_secret}}"}, - {datatype, string} + {datatype, string}, + hidden ]}. %% @doc {auth_bypass, {{auth_bypass}} } , @@ -322,6 +334,32 @@ hidden ]}. +%% == Supercluster aka Multi-bug Support == + +%% @doc IP and port for supercluster members. Specify one member for +%% each line, in which the last token of key is member identifier and +%% value is a PB address of Riak node to connect. +{mapping, "supercluster.member.$member_id", "stanchion.supercluster_members", [ + {datatype, ip}, + {include_default, "bag-A"}, + {commented, {"192.168.0.101", 8087}}, + hidden +]}. + +{translation, + "stanchion.supercluster_members", + fun(Conf) -> + Keys = cuttlefish_variable:fuzzy_matches(["supercluster", "member", "$member_id"], Conf), + case Keys of + [] -> undefined; + [_|_] -> [begin + {Host, Port} = cuttlefish:conf_get( + "supercluster.member." ++ MemberId, Conf), + {MemberId, Host, Port} + end || {Key, MemberId} <- Keys] + end + end +}. %% @doc Cookie for distributed node communication. All nodes in the %% same cluster should use the same cookie or they will not be able to diff --git a/rebar b/rebar deleted file mode 100755 index b2434fe..0000000 Binary files a/rebar and /dev/null differ diff --git a/rebar.config b/rebar.config index 8256094..4f29b25 100644 --- a/rebar.config +++ b/rebar.config @@ -1,28 +1,161 @@ -{sub_dirs, ["rel"]}. +%% -*- erlang -*- +{minimum_otp_vsn, "22.0"}. -{require_otp_vsn, "R16|17"}. +{erl_opts, [ debug_info + , warnings_as_errors + , {parse_transform, lager_transform} + ] +}. -{cover_enabled, true}. +{project_plugins, [ {rebar3_cuttlefish, {git, "https://github.com/TI-Tokyo/rebar3_cuttlefish", {tag, "0.2.1"}}} + ] +}. -{lib_dirs, ["deps", "apps"]}. +{deps, [ {webmachine, {git, "https://github.com/webmachine/webmachine.git", {tag, "1.11.1"}}} + , {mochiweb, "v2.20.1"} + , {riakc, {git, "https://github.com/basho/riak-erlang-client", {tag, "3.0.8+p1"}}} + , {exometer_core, {git, "https://github.com/Feuerlabs/exometer_core", {tag, "v1.5.7"}}} + , {eper, {git, "https://github.com/basho/eper.git", {tag, "0.97.6p1"}}} + , {lager_syslog, {git, "https://github.com/TI-Tokyo/lager_syslog.git", {tag, "3.1.1"}}} + , {rcs_common, {git, "https://github.com/TI-Tokyo/rcs_common.git", {tag, "3.0.4"}}} + ]}. -{erl_opts, [debug_info, - warnings_as_errors, - {parse_transform, lager_transform}, - {platform_define, "^[0-9]+", namespaced_types}]}. +{relx, [ {release, {stanchion, "3.0.0"}, + [ sasl + , stanchion + ] + } + , {overlay, [ {template, "rel/files/advanced.config", "etc/advanced.config"} + , {copy, "rel/files/cert.pem", "etc/cert.pem"} + , {copy, "rel/files/key.pem", "etc/key.pem"} + , {template, "rel/files/stanchion", "usr/bin/stanchion"} + , {template, "rel/files/stanchion-admin", "bin/stanchion-admin"} + , {template, "rel/files/stanchion-debug", "bin/stanchion-debug"} + , {template, "rel/files/lib.sh", "lib/lib.sh"} + , {copy, "rel/files/app_epath.escript", "lib/app_epath.escript"} + , {mkdir, "log"} + ] + } + , {generate_start_script, true} + , {extended_start_script, true} + , {extended_start_script_extensions, + [ {admin, "stanchion-admin"} + , {debug, "stanchion-debug"} + ] + } + ] +}. -{xref_checks, []}. -{xref_queries, [{"(XC - UC) || (XU - X - B)", []}]}. +{profiles, + [ {rel, + [ {relx, + [ {mode, prod} + , {overlay_vars, "rel/vars.config"} + ] + } + ] + } -{reset_after_eunit, true}. + , {dev, + [ {relx, [{mode, dev}]} + ] + } -{deps, [ - {lager, ".*", {git, "git://github.com/basho/lager", {tag, "2.2.0"}}}, - {lager_syslog, ".*", {git, "git://github.com/basho/lager_syslog", {tag, "2.1.1"}}}, - {cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish", {tag, "2.0.4"}}}, - {node_package, ".*", {git, "git://github.com/basho/node_package", {tag, "2.0.3"}}}, - {webmachine, "1.10.*", {git, "git://github.com/basho/webmachine", {tag, "1.10.8"}}}, - {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.1"}}}, - {exometer_core, ".*", {git, "git://github.com/Feuerlabs/exometer_core", {tag, "1.2"}}}, - {eper, ".*", {git, "git://github.com/basho/eper.git", "0.92-basho1"}} - ]}. + , {rpm, + [ {relx, + [ {mode, prod} + , {overlay_vars, "rel/pkg/rpm/vars.config"} + , {extended_start_script_hooks, + [ {post_start, + [{pid, "/run/stanchion/stanchion.pid"}, + {wait_for_process, stanchion_server_sup}] + } + ] + } + ] + } + ] + } + + , {deb, + [ {relx, + [ {mode, prod} + , {overlay_vars, "rel/pkg/deb/vars.config"} + , {extended_start_script_hooks, + [ {post_start, + [{pid, "/run/stanchion/stanchion.pid"}, + {wait_for_process, stanchion_server_sup}] + } + ] + } + ] + } + ] + } + + , {alpine, + [ {relx, + [ {mode, prod} + , {overlay_vars, "rel/pkg/alpine/vars.config"} + , {extended_start_script_hooks, + [ {post_start, + [{pid, "/run/stanchion/stanchion.pid"}, + {wait_for_process, stanchion_server_sup}] + } + ] + } + , {overlay, + [ {template, "rel/pkg/alpine/stanchion.nosu", "usr/bin/stanchion.nosu"} + %% to be renamed to riak-cs in Makefile. We would rather + %% have a special version of riak-cs in pkg/alpine dir, + %% but relx seems to give precedence to to the template + %% entry in the common section. + ] + } + ] + } + ] + } + + , {fbsdng, + [ {relx, + [ {mode, prod} + , {overlay_vars, "rel/pkg/fbsdng/vars.config"} + , {extended_start_script_hooks, + [ {post_start, + [{pid, "/run/stanchion/stanchion.pid"}, + {wait_for_process, stanchion_server_sup}] + } + ] + } + ] + } + ] + } + + %% it's essentially `make rel` tarred + , {osx, + [ {relx, + [ {mode, prod} + , {overlay_vars, "rel/pkg/osx/vars.config"} + ] + } + ] + } + ] +}. + +{cuttlefish, [ {file_name, "stanchion.conf"} + , {disable_bin_scripts, true} + , {schema_discovery, true} + , {schema_order, [ stanchion + , erlang_vm + ]} + ] +}. + +{overrides, + [ {del, eper, [{erl_opts, [warnings_as_errors]}]} + , {del, setup, [{erl_opts, [warnings_as_errors]}]} + ] +}. diff --git a/rebar.docker.config b/rebar.docker.config new file mode 100644 index 0000000..e536db7 --- /dev/null +++ b/rebar.docker.config @@ -0,0 +1,45 @@ +%% -*- erlang -*- +{minimum_otp_vsn, "22.0"}. + +{erl_opts, [ debug_info + , warnings_as_errors + , {parse_transform, lager_transform} + ] +}. + +{deps, [ {webmachine, {git, "https://github.com/webmachine/webmachine.git", {tag, "1.11.1"}}} + , {mochiweb, "v2.20.1"} + , {riakc, {git, "https://github.com/basho/riak-erlang-client", {tag, "3.0.7"}}} + , {exometer_core, {git, "https://github.com/Feuerlabs/exometer_core", {tag, "v1.5.7"}}} + , {eper, {git, "https://github.com/basho/eper.git", "0.97.6p1"}} + , {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}} + , {rcs_common, {git, "https://github.com/TI-Tokyo/rcs_common.git", {tag, "3.0.2"}}} + ]}. + +{relx, [ {release, {stanchion, "3.0"}, + [ sasl + , stanchion + ] + } + , {mode, prod} + , {dev_mode, false} + , {overlay, [ {copy, "rel/files/cert.pem", "etc/cert.pem"} + , {copy, "rel/files/key.pem", "etc/key.pem"} + , {template, "rel/files/stanchion", "usr/sbin/stanchion"} + , {template, "rel/files/stanchion-admin", "usr/sbin/stanchion-admin"} + , {template, "rel/files/stanchion-debug", "usr/sbin/stanchion-debug"} + , {copy, "rel/files/app_epath.sh", "lib/app_epath.sh"} + , {mkdir, "log"} + ] + } + , {generate_start_script, true} + , {extended_start_script, true} + , {sys_config_src, "config/sys.docker.config.src"} + ] +}. + +{overrides, + [ {del, eper, [{erl_opts, [warnings_as_errors]}]} + , {del, setup, [{erl_opts, [warnings_as_errors]}]} + ] +}. diff --git a/rebar.lock b/rebar.lock new file mode 100644 index 0000000..15dbc97 --- /dev/null +++ b/rebar.lock @@ -0,0 +1,70 @@ +{"1.2.0", +[{<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},2}, + {<<"eper">>, + {git,"https://github.com/basho/eper.git", + {ref,"fc165bf4072ecb9ce5648aab86db2baad12122f5"}}, + 0}, + {<<"exometer_core">>, + {git,"https://github.com/Feuerlabs/exometer_core", + {ref,"547f28bc93c7cb3d3f1174fb4c510667a4ebb645"}}, + 0}, + {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.7">>},1}, + {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2}, + {<<"hut">>,{pkg,<<"hut">>,<<"1.2.1">>},1}, + {<<"lager">>, + {git,"https://github.com/erlang-lager/lager.git", + {ref,"22e62f28e5afabe90a6f31bcde367a2b5799fc94"}}, + 1}, + {<<"lager_syslog">>, + {git,"https://github.com/TI-Tokyo/lager_syslog.git", + {ref,"e9117a7e54de35c714fa8d1e6c054854c7c697df"}}, + 0}, + {<<"meck">>, + {git,"https://github.com/eproxus/meck.git", + {ref,"4ecc1ae9089edc6977e8c8c4cd41081513cc5590"}}, + 3}, + {<<"mochiweb">>,{pkg,<<"mochiweb">>,<<"2.20.1">>},0}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},1}, + {<<"protobuffs">>, + {git,"https://github.com/basho/erlang_protobuffs.git", + {ref,"098efad8f85dfe556d64e2cf6ce31f2075808f67"}}, + 2}, + {<<"rcs_common">>, + {git,"https://github.com/TI-Tokyo/rcs_common.git", + {ref,"d0443b2892cbb6e38e4478e14c429a93de26fd2f"}}, + 0}, + {<<"riak_pb">>, + {git,"https://github.com/basho/riak_pb", + {ref,"b7abca90e4c708073021d2c4e18c896f1a10b838"}}, + 1}, + {<<"riakc">>, + {git,"https://github.com/basho/riak-erlang-client", + {ref,"d9fa46335fda4704fda71cbc011ab6a11e50e3f3"}}, + 0}, + {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},1}, + {<<"syslog">>, + {git,"https://github.com/TI-Tokyo/erlang-syslog", + {ref,"14bef508623260a78563d529ccd08e9c259b839e"}}, + 1}, + {<<"webmachine">>, + {git,"https://github.com/webmachine/webmachine.git", + {ref,"92225b82fc702f78cf9d23248023ec841272df80"}}, + 0}]}. +[ +{pkg_hash,[ + {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>}, + {<<"folsom">>, <<"A885F0AEEE4C84270954C88A55A5A473D6B2C7493E32FFDC5765412DD555A951">>}, + {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>}, + {<<"hut">>, <<"08D46679523043424870723923971889E8A34D63B2F946A35B46CF921D1236E7">>}, + {<<"mochiweb">>, <<"E4DBD0ED716F076366ECF62ADA5755A844E1D95C781E8C77DF1D4114BE868CDF">>}, + {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, + {<<"setup">>, <<"1203F4CDA11306C2E34434244576DED0A7BBFB0908D9A572356C809BD0CDF085">>}]}, +{pkg_hash_ext,[ + {<<"bear">>, <<"534217DCE6A719D59E54FB0EB7A367900DBFC5F85757E8C1F94269DF383F6D9B">>}, + {<<"folsom">>, <<"F7B644FC002A75AF00B8BFBD3CC5C2BD955E09A118D2982D9A6C04E5646FF367">>}, + {<<"goldrush">>, <<"99CB4128CFFCB3227581E5D4D803D5413FA643F4EB96523F77D9E6937D994CEB">>}, + {<<"hut">>, <<"953FC447514BAF9CC79FA147D66469243C94DFA1593779614E070C692D0BF0F3">>}, + {<<"mochiweb">>, <<"D1AEEE7870470D2FA9EAE0B3D5AB6C33801AA2D82B10E9DADE885C5C921B36AA">>}, + {<<"parse_trans">>, <<"17EF63ABDE837AD30680EA7F857DD9E7CED9476CDD7B0394432AF4BFC241B960">>}, + {<<"setup">>, <<"7D6AAF5281D0B0C40980E128F9DC410DACD03799A8577201D4C8B43E7F97509A">>}]} +]. diff --git a/rebar3 b/rebar3 new file mode 100755 index 0000000..c1597bb Binary files /dev/null and b/rebar3 differ diff --git a/rel/dev_vars.config b/rel/dev_vars.config index 12e8ea0..c0c8f8a 100644 --- a/rel/dev_vars.config +++ b/rel/dev_vars.config @@ -1 +1,34 @@ +%% -*- erlang -*- + {devrel, true}. + +{rel_vsn, "{{release_version}}"}. + +%% Platform-specific installation paths +{platform_base_dir, "@PLATFORM_BASE_DIR@"}. +{platform_bin_dir, "{{platform_base_dir}}/bin"}. +{platform_data_dir, "{{platform_base_dir}}/data"}. +{platform_etc_dir, "{{platform_base_dir}}/etc"}. +{platform_lib_dir, "{{platform_base_dir}}/lib"}. +{platform_log_dir, "{{platform_base_dir}}/log"}. +{platform_gen_dir, "{{platform_base_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. +%% +%% etc/app.config +%% +{stanchion_version, 030000}. +{stanchion_ip, "127.0.0.1"}. +{stanchion_port, 8085}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. + +%% +%% etc/vm.args +%% +{node, "stanchion@127.0.0.1"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +%% lager +{console_log_default, file}. diff --git a/rel/files/advanced.config b/rel/files/advanced.config index a533026..e649a76 100644 --- a/rel/files/advanced.config +++ b/rel/files/advanced.config @@ -1 +1,9 @@ -[{stanchion, []}]. +[{stanchion, + [] + }, + {setup, + [{data_dir, "{{platform_data_dir}}"}, + {log_dir, "{{platform_log_dir}}"} + ] + } +]. diff --git a/rel/files/app_epath.escript b/rel/files/app_epath.escript new file mode 100644 index 0000000..f84608c --- /dev/null +++ b/rel/files/app_epath.escript @@ -0,0 +1,9 @@ +-module(app_epath). +-export([main/1]). + +main([AdvancedConfig]) -> + {ok, [AA]} = file:consult(AdvancedConfig), + [p(A) || A <- AA]. + +p({App, CC}) -> + [io:format("~s ~s ~9999p\n", [App, K, V]) || {K, V} <- CC]. diff --git a/rel/files/lib.sh b/rel/files/lib.sh new file mode 100644 index 0000000..2518f3e --- /dev/null +++ b/rel/files/lib.sh @@ -0,0 +1,87 @@ +fmt() { + echo -n "[[" + local n=$# + if [ $n -ne 0 ]; then + echo -n "\"$1\"" + shift + n=$((n-1)) + fi + while [ $n -ne 0 ]; do + echo -n ", \"$1\"" + shift + n=$((n-1)) + done + echo "]]" +} + +rpc() { + local mod=$1 + local fun=$2 + shift 2 + if [ $# -gt 0 ]; then + "${PLATFORM_BIN_DIR}/stanchion" rpc $mod $fun `fmt "$@"` + else + "${PLATFORM_BIN_DIR}/stanchion" rpc $mod $fun "[[]]" + fi +} + + +## Example usage: +# +# #!/bin/sh +# +# # Load the functions +# . path/to/app_epath.sh +# +# # Build the path info +# epaths=`make_app_epaths path/to/app.config` +# +# # View all of the settings. Quotes are important. +# echo "$epaths" +# +# # Grep around in the paths for various items. +# echo "$epaths" | grep 'riak_core ring_creation_size' +# echo "$epaths" | grep "lager handlers lager_file_backend" | grep info +# +# # Use the epath function to get directly at settings +# epath 'riak_core ring_creation_size' "$epaths" +# epath 'riak_core platform_bin_dir' "$epaths" +# epath 'riak_kv storage_backend' "$epaths" +# +# # Use epath to view all of the riak_core settings +# epath riak_core +# +## End example + +## Here is a command you can put in your path for general cli use. +# +# #!/bin/sh +# +# # $1 quoted eterm path for search: 'riak_core ring_creation_size' +# # $2 path/to/app.config +# +# . path/to/app_epath.sh +# +# epaths=`make_app_epaths "$2"` +# epath "$1" "$epaths" +# +## End epath command + +# make_app_epaths takes a path to an app.config file as its argument and +# and returns (prints) a flattened text structure of configuration settings. + +make_app_epaths () { + ERTS_VER=$(cd ${PLATFORM_BASE_DIR} && ls -d erts-*) + ERTS_PATH="${PLATFORM_BASE_DIR}/$ERTS_VER/bin" + $ERTS_PATH/escript $stanchion_lib_dir/app_epath.escript "$1" +} + +epath () { + # arg1 - a pattern to search for + # arg2 - output of make_app_epaths, passed in quoted + # output - search of arg2 for arg1, trimming arg1 from the beginning + # Note: there may be multiple lines of output. + pat=$1 + shift + echo "$*" | grep "$pat " | sed "s/^${pat} *//" +} diff --git a/rel/files/stanchion b/rel/files/stanchion new file mode 100755 index 0000000..7bc66c8 --- /dev/null +++ b/rel/files/stanchion @@ -0,0 +1,76 @@ +#!/bin/bash + +## ------------------------------------------------------------------- +## +## stanchion: Stanchion launcher +## +## Copyright (c) 2014 Basho Technologies, Inc., +## 2021 TI Tokyo. All Rights Reserved. +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## ------------------------------------------------------------------- + +RUNNER_GEN_DIR={{platform_gen_dir}} +RELX_STANCHION={{platform_bin_dir}}/stanchion +export PID_DIR={{pid_dir}} +export RUNNER_LOG_DIR={{platform_log_dir}} + +mkdir -p $PID_DIR +chown stanchion:stanchion $PID_DIR + +# centos7-based distros have a su that contacts pam and prints the "Last logged in" message +if [ "`cat /etc/redhat-release 2>&1`" = "CentOS Stream release 7" ] || + [ "`cat /etc/system-release 2>&1`" = "Amazon Linux release 2 (Karoo)" ]; then + COPTION="--session-command" +else + COPTION="-c" +fi + +function maybe_su { + if [[ $EUID -ne 0 ]]; then + $* + else + # if we are executing an admin command that spins up a + # (hidden) node to then execute custom erlang code via -eval, + # we need to cd to a dir containing the erlang cookie + # (previously implicitly done by su -, which option we have + # removed in order to allow any env vars to be available for + # the ultimate invocation of riak/riak-cs/stanchion) + cd "{{platform_base_dir}}" + # freebsd su is fairly limited, so: + mkdir -p "$RUNNER_GEN_DIR" + chown stanchion:stanchion "$RUNNER_GEN_DIR" + f=`mktemp "$RUNNER_GEN_DIR"/su_piggy-XXXXXXX` + cat >"$f" <&2; } + +mkdir_or_die () { + # If the dir already exists, just return + [ -d "$1" ] && return + + mkdir -p "$1" + if [ 0 -ne $? ]; then + echoerr "Error creating stanchion-debug directories. Aborting." + echoerr "$1" + exit 1 + fi +} + +dump () { + # first argument is the filename to hold the command output + out=$1 + shift + + # next argument is the base command to execute. skip dump if not available. + [ -z "`command -v $1`" ] && return + + # execute the rest of the arguments as the command + $* >> "$out" 2>&1 + + # grab the return value + retval=$? + + # put the command and the retval in the .info/$out file to aid automation. + # note: this will miss some escaping for, e.g., find, but it's not critical. + # get command that was run with `head -1 .info/$out` + # get return value from command with `tail -1 .info/$out` + echo "$*" > .info/"$out" + echo $retval >> .info/"$out" + + if [ 0 -eq $retval ]; then + printf '.' 1>&2 + else + if [ $verbose_output -gt 0 ]; then + echoerr 'Command '\'"$*"\'' failed. Continuing.' + else + printf 'E' 1>&2 + fi + fi + + return $retval +} + +usage () { +cat << 'EOF' +Usage: stanchion-debug [-c] [-l] [-r] [-s] [-e] [-v] [FILENAME | -] + +-c, --cfgs Gather Stanchion configuration information (includes everything + in platform_etc_dir). + Please see the Privacy Note below. + --ssl-certs Do not skip the capture of *potentially private* SSL + certificates. + By default files in the platform_etc_dir with the following + extensions are not included in the stanchion-debug archive: + .pfx, .p12, .csr, .sst, .sto, .stl, .pem, .key. + Please see the Privacy Note below. +-l, --logs Gather Stanchion logs. +-r, --riakcmds Gather Stanchion information and command output. +-s, --syscmds Gather general system commands. +-e, --extracmds Gather extra command output. These commands are too intense + to run on all nodes in a cluster but appropriate for a + single node. +-v, --verbose Print verbose failure messages to stderr + --log-mtime Determine last modified time as n*24 hours, which + stanchion-debug gathers newer logs than the time. + (default: 14) + --skip-accesslog Exclude access.log from logs stanchion-debug gathers. +FILENAME Output filename for the tar.gz archive. Use - to specify stdout. + +Defaults: Get configs, logs, stanchion commands, and system commands. + Output in current directory to NODE_NAME-stanchion-debug.tar.gz or + HOSTNAME-stanchion-debug.tar.gz if NODE_NAME cannot be found. + +Privacy Note: When the -c flag is set (included in the defaults) every file in + the specified in Riak's `platform_etc_dir` will be copied into the + generated debug archive with the exceptions noted above. If there + are additional files that you wish to keep out of the archive, the + enviroment variable RIAK_EXCLUDE may be filled with a space + separated list of file patterns that will be passed into a `find + -name`. Any matches will be excluded from the generated archive. + E.g. `RIAK_EXCLUDE="'*.key' mySecretFile.txt" stanchion-debug` +EOF +exit +} + +### +### Set up variables +### + +is_relative() { + if [ "${1#/}" = "$1" ]; then + return 0 + else + return 1 + fi +} +PLATFORM_BASE_DIR={{platform_base_dir}} +PLATFORM_BASE_DIR=${PLATFORM_BASE_DIR:-$(cd $(dirname "$0")/.. && pwd -P)} +stanchion_base_dir=$PLATFORM_BASE_DIR +if is_relative "{{platform_bin_dir}}"; then + stanchion_bin_dir="$PLATFORM_BASE_DIR/{{platform_bin_dir}}" +else + stanchion_bin_dir="{{platform_bin_dir}}" +fi +if is_relative "{{platform_etc_dir}}"; then + stanchion_etc_dir="$PLATFORM_BASE_DIR/{{platform_etc_dir}}" +else + stanchion_etc_dir="{{platform_etc_dir}}" +fi +if is_relative "{{platform_log_dir}}"; then + stanchion_log_dir="$PLATFORM_BASE_DIR/{{platform_log_dir}}" +else + stanchion_log_dir="{{platform_log_dir}}" +fi +if is_relative "{{platform_lib_dir}}"; then + stanchion_lib_dir="$PLATFORM_BASE_DIR/{{platform_lib_dir}}" +else + stanchion_lib_dir="{{platform_lib_dir}}" +fi +if is_relative "{{platform_gen_dir}}"; then + stanchion_gen_dir="$PLATFORM_BASE_DIR/{{platform_gen_dir}}" +else + stanchion_gen_dir="{{platform_gen_dir}}" +fi + +cuttlefish_conf_dir=${stanchion_gen_dir}/generated.conf + +get_cfgs=0 +get_ssl_certs=0 +get_logs=0 +get_riakcmds=0 +get_syscmds=0 +get_extracmds=0 +skip_accesslog=0 +verbose_output=0 + +log_mtime=14 + +### +### Parse options +### + +while [ -n "$1" ]; do + case "$1" in + -h|--help) + usage + ;; + -c|--cfgs) + get_cfgs=1 + ;; + --ssl-certs) + get_ssl_certs=1 + ;; + -l|--logs) + get_logs=1 + ;; + -r|--riakcmds) + get_riakcmds=1 + ;; + -s|--syscmds) + get_syscmds=1 + ;; + -e|--extracmds) + get_extracmds=1 + ;; + --log-mtime) + shift + log_mtime=$1 + ;; + --skip-accesslog) + skip_accesslog=1 + ;; + -v|--verbose) + verbose_output=`expr 1 + $verbose_output` + ;; + -) + # If truly specifying stdout as the output file, it should be last + if [ $# -gt 1 ]; then + echoerr "Trailing options following filename $1. Aborting." + echoerr "See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + + outfile="-" + ;; + *) + # Shouldn't get here until the last option, the output filename. + if [ '-' = "$outfile" ]; then + echoerr "Filename $1 given but stdout, -, already specified." + echoerr "Aborting. See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + + # The filename shouldn't start with a '-'. The single character '-' + # is handled as a special case above. + if [ '-' = `echo "$1" | cut -c1` ]; then + echoerr "Unrecognized option $1. Aborting" + echoerr "See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + + if [ $# -gt 1 ]; then + echoerr "Trailing options following filename $1. Aborting." + echoerr "See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + + outfile="$1" + ;; + esac + shift +done + +cd $PLATFORM_BASE_DIR + +### +### Finish setting up variables and overrides +### + +if [ 0 -eq $(( $get_cfgs + $get_logs + $get_riakcmds + $get_syscmds + $get_extracmds )) ]; then + # Nothing specific was requested, so get everything except extracmds + get_cfgs=1 + get_logs=1 + get_riakcmds=1 + get_syscmds=1 + get_extracmds=0 +fi + +if [ 0 -eq $get_cfgs -a 1 -eq $get_ssl_certs ]; then + echoerr "The --ssl-certs option is meaningless without --cfg. Ignoring." + get_ssl_certs=0 +fi + +if [ 0 -ne $(( $get_cfgs + $get_logs + $get_riakcmds + $get_extracmds )) ]; then + # Information specific to Riak requested. Need app_epath.sh and must have + # valid base and etc paths defined. + + . $stanchion_lib_dir/lib.sh + + # Allow overriding stanchion_base_dir and stanchion_etc_dir from the environment + if [ -n "$stanchion_base_dir" ]; then + stanchion_base_dir="$stanchion_base_dir" + if [ "/" != "`echo "$stanchion_base_dir" | cut -c1`" ]; then + echoerr "Stanchion base directory should be an absolute path." + echoerr "$stanchion_base_dir" + echoerr "See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + fi + + if [ -n "$stanchion_bin_dir" ]; then + stanchion_bin_dir="$stanchion_bin_dir" + if [ "/" != "`echo "$stanchion_bin_dir" | cut -c1`" ]; then + echoerr "Stanchion bin directory should be an absolute path." + echoerr "$stanchion_bin_dir" + echoerr "See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + fi + + if [ -n "$stanchion_etc_dir" ]; then + stanchion_etc_dir="$stanchion_etc_dir" + if [ "/" != "`echo "$stanchion_etc_dir" | cut -c1`" ]; then + echoerr "Stanchion etc directory should be an absolute path." + echoerr "$stanchion_etc_dir" + echoerr "See 'stanchion-debug -h' and manpage for help." + exit 1 + fi + fi +fi + +#prefer vm.args if it exists +if [ -f "${stanchion_etc_dir}/vm.args" ]; then + vmargs="${stanchion_etc_dir}/vm.args" +elif [ -d "${cuttlefish_conf_dir}" ]; then + vmargs=`ls -t ${cuttlefish_conf_dir}/vm.*.args | head -1` +fi + + +if [ -f "${vmargs}" ]; then + node_name="`egrep '^\-s?name' "${vmargs}" 2>/dev/null | cut -d ' ' -f 2`" +fi + +if [ -z "$node_name" ]; then + # Couldn't figure out node name. Fallback to hostname. + node_name="`hostname`" +fi + +start_dir="$TMPDIR" + +if [ -z "$start_dir" ]; then + start_dir=/tmp +fi + +# Strip any trailing slash from TMPDIR +start_dir="`echo $start_dir | sed 's#/$##'`" + +debug_dir="${node_name}-stanchion-debug" + +if [ -d "${start_dir}"/"${debug_dir}" ]; then + echoerr "Temporary directory already exists. Aborting." + echoerr "${start_dir}"/"${debug_dir}" + exit 1 +fi + +if [ -z "$outfile" ]; then + # If output file not specified, output to the default + outfile="`pwd`"/"${debug_dir}".tar.gz +fi + +if [ '-' != "$outfile" ] && [ -f "$outfile" ]; then + echoerr "Output file already exists. Aborting." + echoerr "$outfile" + exit 1 +fi + +### +### Gather system commands +### + +if [ 1 -eq $get_syscmds ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info + cd "${start_dir}"/"${debug_dir}"/commands + + # System info + dump date date + dump w w + dump last last + dump hostname hostname + dump uname uname -a + dump lsb_release lsb_release + dump ps ps aux + dump vmstat vmstat 1 5 + dump free free -m + dump df df + dump df_i df -i + dump dmesg dmesg + dump mount mount + dump sysctl sysctl -a + dump rpm rpm -qa + dump dpkg dpkg -l + dump pkg_info pkg_info + dump sestatus sestatus -v + dump ifconfig ifconfig -a + dump netstat_i netstat -i + dump netstat_an netstat -an + dump netstat_rn netstat -rn + dump pfctl_rules pfctl -s rules + dump pfctl_nat pfctl -s nat + dump zfs_list zfs list + dump zpool_list zpool list + + # If swapctl exists, prefer it over swapon + if [ -n "`command -v swapctl`" ]; then + dump swapctl swapctl -s + else + dump swapon swapon -s + fi + + BLOCKDEV=/sbin/blockdev + if [ -e $BLOCKDEV ]; then + for mount_point in `mount | egrep '^/' | awk '{print $1}'`; do + flat_point=`echo $mount_point | sed 's:/:_:g'` + dump blockdev.$flat_point $BLOCKDEV --getra $mount_point + done + else + dump blockdev._ echo $BLOCKDEV is not available + fi + + # Running iptables commands if the module is not loaded can automatically + # load them. This is rarely desired and can even cause connectivity + # problems if, e.g., nf_conntrack gets autoloaded and autoenabled. + if [ -n "`command -v lsmod`" ]; then + if [ -n "`lsmod 2>/dev/null | awk '{print $1}' | grep iptable_filter`" ]; then + dump iptables_rules iptables -n -L + else + dump iptables_rules echo "iptables module not loaded" + fi + + if [ -n "`lsmod 2>/dev/null | awk '{print $1}' | grep nf_conntrack`" ]; then + dump iptables_nat iptables -t nat -n -L + else + dump iptables_nat echo "nf_conntrack module not loaded" + fi + fi + + if [ -f /proc/diskstats ]; then + # Linux iostat + dump iostat_linux iostat -mx 1 5 + elif [ -d /proc ]; then + # No diskstats, but proc, probably Solaris or SmartOS + dump iostat_smartos iostat -xnz 1 5 + else + # BSD style iostat + dump iostat_bsd iostat -dIw 1 -c 5 + fi + + # Dump files + [ -f /etc/release ] && dump release cat /etc/release + [ -f /etc/redhat-release ] && dump redhat_release cat /etc/redhat-release + [ -f /etc/debian_version ] && dump debian_version cat /etc/debian_version + [ -f /etc/security/limits.conf ] && dump limits.conf cat /etc/security/limits.conf + [ -f /var/log/messages ] && dump messages cat /var/log/messages + [ -f /var/log/syslog ] && dump syslog cat /var/log/syslog + [ -f /var/log/kern.log ] && dump kern.log cat /var/log/kern.log + [ -f /proc/diskstats ] && dump diskstats cat /proc/diskstats + [ -f /proc/cpuinfo ] && dump cpuinfo cat /proc/cpuinfo + [ -f /proc/meminfo ] && dump meminfo cat /proc/meminfo + + # Dump directories and finds + [ -d /dev/disk/by-id ] && dump disk_by_id ls -l /dev/disk/by-id + [ -d /sys/block ] && dump schedulers find /sys/block/ -type l -print -exec cat {}/queue/scheduler \; + [ -d /proc/net/bonding ] && dump bonding find /proc/net/bonding/ -type f -print -exec cat {} \; + [ -d /sys/class/net ] && dump rx_crc_errors find /sys/class/net/ -type l -print -exec cat {}/statistics/rx_crc_errors \; + + # A bit more complicated, but let's get all of limits.d if it's there + if [ -d /etc/security/limits.d ]; then + # check to ensure there is at least something to get + ls -1 /etc/security/limits.d | grep -q ".conf" + if [ 0 -eq $? ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/limits.d + + # Mirror the directory, only copying files that match the pattern + cd /etc/security/limits.d + find . -type f -name '*.conf' -exec sh -c ' + mkdir -p "$0/${1%/*}"; + cp "$1" "$0/$1" + ' "${start_dir}"/"${debug_dir}"/commands/limits.d {} \; + fi + fi +fi + +### +### Gather Stanchion commands and info +### + +if [ 1 -eq $get_riakcmds ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info + cd "${start_dir}"/"${debug_dir}"/commands + + dump stanchion_ping "$stanchion_bin_dir"/stanchion ping + dump stanchion_version "$stanchion_bin_dir"/stanchion-admin version + dump stanchion_getpid "$stanchion_bin_dir"/stanchion pid + dump stanchion_admin_status "$stanchion_bin_dir"/stanchion-admin status +fi + +### +### Gather Stanchion logs +### + +if [ 1 -eq $get_logs ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/.info + cd "${start_dir}"/"${debug_dir}"/logs + + # if not already made, make a flat, searchable version of the app.config + if [ -z "$riak_epaths" ]; then + #prefer app.config if it exists + if [ -f "${stanchion_etc_dir}/app.config" ]; then + appconfig="${stanchion_etc_dir}/app.config" + riak_epaths=`make_app_epaths "${appconfig}"` + elif [ -d "${cuttlefish_conf_dir}" ]; then + appconfig=`ls -t ${cuttlefish_conf_dir}/app.*.config | head -1` + riak_epaths=`make_app_epaths "${appconfig}"` + fi + fi + + # grab a listing of the log directory to show if there are crash dumps + dump ls_log_dir ls -lhR $stanchion_log_dir + + # Get any logs in the platform_log_dir + if [ -d "${stanchion_log_dir}" ]; then + # check to ensure there is at least something to get + ls -1 "${stanchion_log_dir}" | grep -q "log" + if [ 0 -eq $? ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/platform_log_dir + + # Mirror the directory, only copying files that match the pattern + cd "$stanchion_log_dir" + find . -type f -mtime -$log_mtime -exec sh -c ' + mkdir -p "$0/${1%/*}"; + cp -p "$1" "$0/$1" + ' "${start_dir}"/"${debug_dir}"/logs/platform_log_dir {} \; + + # Remove access logs + if [ 1 -eq $skip_accesslog ]; then + find "${start_dir}"/"${debug_dir}"/logs/platform_log_dir -type f -name 'access.log*' -exec rm {} \; + fi + fi + fi + + # Lager info and error files + new_format_lager_files="`epath 'lager handlers lager_file_backend file' "$riak_epaths" | sed -e 's/^"//' -e 's/".*$//'`" + if [ -z "$riak_epaths" ]; then + lager_files="" + elif [ -z "$new_format_lager_files" ]; then + lager_files="`epath 'lager handlers lager_file_backend' "$riak_epaths" | cut -d' ' -f 1 | sed -e 's/^"//' -e 's/".*$//'`" + else + lager_files=`echo "$new_format_lager_files"` + fi + lager_num=0 + for lager_path in $lager_files; do + # Get lager logs if they weren't in platform_log_dir + if [ -n "$lager_path" ]; then + if [ '/' != `echo "$lager_path" | cut -c1` ]; then + # relative path. prepend base dir + lager_path="$stanchion_base_dir"/"$lager_path" + fi + + lager_file="`echo $lager_path | awk -F/ '{print $NF}'`" + lager_dir="`echo $lager_path | awk -F/$lager_file '{print $1}'|sed 's|/\./|/|'`" + if [ "$stanchion_log_dir" != "$lager_dir" ]; then + ls -1 "${lager_dir}" | grep -q "${lager_file}" + if [ 0 -eq $? ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/lager_${lager_num} + find "${lager_dir}" -mtime -$log_mtime -name "${lager_file}*" \ + -exec cp -p {} "${start_dir}"/"${debug_dir}"/logs/lager_"${lager_num}"/ \; + fi + fi + lager_num=`expr $lager_num + 1` + fi + done +fi + +### +### Gather Stanchion configuration +### + +if [ 1 -eq $get_cfgs ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/config/.info + + # Capture the needed files from the `stanchion_etc_dir` directory. + cd $stanchion_etc_dir + + # Convert the list of file blobs to a list of `"! -name `. + # For example if `RIAK_EXCLUDE="*.key mySecretFile"`, this will set + # `exclude="! -name *.key ! -name mySecretFile"`. + exclude="" + for i in `echo $RIAK_EXCLUDE`; do + exclude="${exclude}! -name $i " + done + if [ 0 -eq $get_ssl_certs ]; then + for i in `echo "'*.pfx' '*.p12' '*.csr' '*.sst' '*.sto' '*.stl' '*.pem' '*.key'"`; do + exclude="${exclude}! -name $i " + done + fi + + # Compose the `find` command that will exclude the above list of files, + # being aware that an empty `\( \)` will cause a failure in the `find`. + if [ -z "$exclude" ]; then + run="find . -type f -exec sh -c ' + mkdir -p \"\$0/\${1%/*}\"; + cp \"\$1\" \"\$0/\$1\" + ' \"\${start_dir}\"/\"\${debug_dir}\"/config {} \;" + else + run="find . -type f \\( $exclude \\) -exec sh -c ' + mkdir -p \"\$0/\${1%/*}\"; + cp \"\$1\" \"\$0/\$1\" + ' \"\${start_dir}\"/\"\${debug_dir}\"/config {} \;" + fi + eval $run + + # Copy the generated configurations into the archive. + cd "${start_dir}"/"${debug_dir}"/config + + # Use dump to execute the copy. This will provide a progress dot and + # capture any error messages. + dump cp_generated_cfg cp -R "$cuttlefish_conf_dir" . + + # If the copy succeeded, then the output will be empty and it is unneeded. + if [ 0 -eq $? ]; then + rm -f cp_generated_cfg + fi +fi + +### +### Produce the output file +### + +# One last sanity check before transferring or removing anything +cd "${start_dir}" +if [ -z "$debug_dir" ] || [ ! -d "$debug_dir" ]; then + echoerr "Couldn't find ${start_dir}/${debug_dir}. Aborting" + exit 1 +fi + +if [ '-' = "$outfile" ]; then + # So we don't get a file literally named - + tar zcf - "${debug_dir}" +else + tar zcf "$outfile" "${debug_dir}" + + # provide some indication of the output filename + printf " $outfile" 1>&2 +fi +rm -rf "${debug_dir}" + +# keep things looking pretty +echoerr "" diff --git a/rel/pkg/.gitignore b/rel/pkg/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/rel/pkg/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/rel/pkg/Makefile b/rel/pkg/Makefile new file mode 100644 index 0000000..a894277 --- /dev/null +++ b/rel/pkg/Makefile @@ -0,0 +1,92 @@ +## +## Export all variables to sub-invocation +## +export + +OS = $(shell uname -s) +ERLANG_BIN ?= $(shell dirname $(shell which erl)) + +## +## Support RPM and Debian based linux systems +## +ifeq ($(OS),Linux) +ARCH = $(shell uname -m) +ISAMZ = $(shell cat /etc/system-release 2> /dev/null) +ISRPM = $(shell cat /etc/redhat-release 2> /dev/null) +ISDEB = $(shell cat /etc/debian_version 2> /dev/null) +ISSLES = $(shell cat /etc/SuSE-release 2> /dev/null) +ifneq ($(ISAMZ),) +OSNAME = Amazon +PKGERDIR = rpm +BUILDDIR = rpmbuild +else +ifneq ($(ISRPM),) +OSNAME = RedHat +PKGERDIR = rpm +BUILDDIR = rpmbuild +else +ifneq ($(ISDEB),) +OSNAME = Debian +PKGERDIR = deb +BUILDDIR = debuild +else +ifneq ($(ISSLES),) +OSNAME = SLES +PKGERDIR = rpm +BUILDDIR = rpmbuild +endif # SLES +endif # deb +endif # rpm +endif # amazon +endif # linux + +ifeq ($(OS),Darwin) # OSX +OSNAME = OSX +ARCH = $(shell file `which erlc` | grep -c x86_64 2> /dev/null | awk \ + '{if ($$1 == "0") {print "i386"} else {print "x86_64"}}') +PKGERDIR = osx +BUILDDIR = osxbuild +endif + +ifeq ($(OS),FreeBSD) +OSNAME = FreeBSD +ARCH = $(shell uname -m) +BUILDDIR = fbsdbuild +PKGERDIR = fbsdng +endif + +ifeq ($(OS),SunOS) # Solaris flavors +KERNELVER = $(shell uname -v | grep -c joyent 2> /dev/null) +ARCH = $(shell file `which erlc` | grep -c 64-bit 2> /dev/null | awk \ + '{if ($$1 == "0") {print "i386"} else {print "x86_64"}}') + +ifneq ($(KERNELVER),0) # SmartOS +OSNAME = SmartOS +PKGERDIR = smartos +BUILDDIR = smartosbuild +else # Solaris / OmniOS +DISTRO = $(shell head -1 /etc/release|awk \ + '{if ($$1 == "OmniOS") {print $$1} else {print "Solaris"}}') +OSNAME = ${DISTRO} +PKGERDIR = solaris +BUILDDIR = solarisbuild +endif + +endif + +DATE = $(shell date +%Y-%m-%d) + +# Default the package build version to 1 if not already set +PKG_BUILD ?= 1 + +.PHONY: ostype varcheck + +## Call platform dependent makefile +ostype: varcheck + $(if $(PKGERDIR),,$(error "Operating system '$(OS)' not supported by node_package")) + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/stanchion-$(PKG_ID) -f $(BASE_DIR)/rel/pkg/$(PKGERDIR)/Makefile + +## Check required settings before continuing +varcheck: + $(if $(PKG_VERSION),,$(error "Variable PKG_VERSION must be set and exported, see basho/node_package readme")) + $(if $(PKG_ID),,$(error "Variable PKG_ID must be set and exported, see basho/node_package readme")) diff --git a/rel/pkg/alpine/stanchion.nosu b/rel/pkg/alpine/stanchion.nosu new file mode 100755 index 0000000..59277b9 --- /dev/null +++ b/rel/pkg/alpine/stanchion.nosu @@ -0,0 +1,26 @@ +#!/bin/bash + +RUNNER_GEN_DIR={{platform_gen_dir}} +RELX_RIAK={{platform_bin_dir}}/stanchion +export PID_DIR={{pid_dir}} + +PID_FILE=$PID_DIR/stanchion.pid + +mkdir -p $PID_DIR + +case "$1" in + start) + $RELX_RIAK $* -pa {{platform_patch_dir}} + test -r $PID_FILE && exit 0 + ;; + console|foreground) + $RELX_RIAK $* -pa {{platform_patch_dir}} + ;; + stop) + $RELX_RIAK $* \ + && rm -f $PID_FILE + ;; + *) + $RELX_RIAK $* + ;; +esac diff --git a/rel/pkg/alpine/vars.config b/rel/pkg/alpine/vars.config new file mode 100644 index 0000000..7f27518 --- /dev/null +++ b/rel/pkg/alpine/vars.config @@ -0,0 +1,45 @@ +%% -*- mode: erlang -*- +%% ex: ft=erlang ts=4 sw=4 et + +{rel_vsn, "{{release_version}}"}. + +%% Platform-specific installation paths +{platform_bin_dir, "/usr/lib/stanchion/bin"}. +{platform_data_dir, "/var/lib/stanchion"}. +{platform_etc_dir, "/etc/stanchion"}. +{platform_base_dir, "/usr/lib/stanchion"}. +{platform_lib_dir, "/usr/lib/stanchion/lib"}. +{platform_log_dir, "/var/log/stanchion"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +%% +%% etc/app.config +%% +{stanchion_version, 030000}. +{stanchion_ip, "127.0.0.1"}. +{stanchion_port, 8085}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. + +%% +%% etc/vm.args +%% +{node, "stanchion@127.0.0.1"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/run/stanchion"}. + +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. + +%% +%% cuttlefish +%% +{cuttlefish, "on"}. +{cuttlefish_conf, "stanchion.conf"}. + +%% lager +{console_log_default, file}. diff --git a/rel/pkg/deb/Makefile b/rel/pkg/deb/Makefile new file mode 100644 index 0000000..38f8c11 --- /dev/null +++ b/rel/pkg/deb/Makefile @@ -0,0 +1,29 @@ +export + +TAR_VERSION = $(shell git describe --tags | sed -e 's/\([0-9.]*\-[0-9]*\)-.*/\1/') + +DEBEMAIL = $(shell git config user.email) + +default: + mkdir -p $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/debian + cp -R $(BASE_DIR)/rel/pkg/deb/debian/* $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/debian + + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/ + ln -sf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz $(BASE_DIR)/rel/pkg/out/$(TAR_VERSION).orig.tar.gz + + cd $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/; \ + echo $(shell pwd); \ + dch --create --package stanchion -v "$(PKG_VERSION)" \ + "Build from $(PKG_ID)";\ + debuild --prepend-path=$(ERLANG_BIN) \ + -e REVISION=$(PKG_VERSION) \ + -e RELEASE=$(PKG_BUILD) \ + -e REBAR=$(REBAR) \ + -i -uc -us -b + + mkdir -p $(BASE_DIR)/rel/pkg/out/packages + cd $(BASE_DIR)/rel/pkg/out && mv *.deb ../out/packages + cd $(BASE_DIR)/rel/pkg/out/packages && \ + for debfile in *.deb; do \ + sha256sum $${debfile} > $${debfile}.sha \ + ; done diff --git a/rel/pkg/deb/debian/compat b/rel/pkg/deb/debian/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/rel/pkg/deb/debian/compat @@ -0,0 +1 @@ +10 diff --git a/rel/pkg/deb/debian/control b/rel/pkg/deb/debian/control new file mode 100644 index 0000000..a86aa76 --- /dev/null +++ b/rel/pkg/deb/debian/control @@ -0,0 +1,13 @@ +Source: stanchion +Section: net +Priority: optional +Maintainer: Andrei Zavada +Build-Depends: debhelper (>= 7) +Standards-Version: 4.6.0.1 +Homepage: https://tiot.jp + +Package: stanchion +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, adduser, logrotate, erlang-base +Homepage: https://tiot.jp +Description: Serialization service for Riak CS diff --git a/rel/pkg/deb/debian/copyright b/rel/pkg/deb/debian/copyright new file mode 100644 index 0000000..97d412c --- /dev/null +++ b/rel/pkg/deb/debian/copyright @@ -0,0 +1,7 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: stanchion +Upstream-Contact: Andrei Zavada + +Files: * +Copyright: 2012-16 Basho Technologies, Inc; 2021 TI Tokyo +License: Apache-2.0 diff --git a/rel/pkg/deb/debian/dirs b/rel/pkg/deb/debian/dirs new file mode 100644 index 0000000..ff2baad --- /dev/null +++ b/rel/pkg/deb/debian/dirs @@ -0,0 +1,3 @@ +etc/logrotate.d +var/lib/stanchion +var/log/stanchion diff --git a/rel/pkg/deb/debian/install b/rel/pkg/deb/debian/install new file mode 100644 index 0000000..cc50c19 --- /dev/null +++ b/rel/pkg/deb/debian/install @@ -0,0 +1,8 @@ +rel/stanchion/lib usr/lib/stanchion +rel/stanchion/share usr/lib/stanchion +rel/stanchion/erts* usr/lib/stanchion +rel/stanchion/releases usr/lib/stanchion +rel/stanchion/bin/stanchion-* usr/sbin +rel/stanchion/usr/bin/stanchion usr/sbin +rel/stanchion/bin usr/lib/stanchion +rel/stanchion/etc/* etc/stanchion diff --git a/rel/pkg/deb/debian/postinst b/rel/pkg/deb/debian/postinst new file mode 100755 index 0000000..288818b --- /dev/null +++ b/rel/pkg/deb/debian/postinst @@ -0,0 +1,43 @@ +#!/bin/sh +# postinst script for stanchion +# +# see: dh_installdeb(1) + +set -e + +# create group +if ! getent group stanchion >/dev/null; then + addgroup --system stanchion +fi + +# create user +if ! getent passwd stanchion >/dev/null; then + adduser --ingroup stanchion \ + --home /var/lib/stanchion \ + --disabled-password \ + --system --shell /bin/bash --no-create-home \ + --gecos "Riak CS Database" stanchion +fi + +chown stanchion:stanchion /var/log/stanchion /etc/stanchion /var/lib/stanchion +chmod 750 /var/log/stanchion /etc/stanchion /var/lib/stanchion + +case "$1" in + configure) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/rel/pkg/deb/debian/postrm b/rel/pkg/deb/debian/postrm new file mode 100755 index 0000000..24799d8 --- /dev/null +++ b/rel/pkg/deb/debian/postrm @@ -0,0 +1,64 @@ +#!/bin/sh +# postrm script for stanchion +# +# see: dh_installdeb(1) + + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + +set -e + +case "$1" in + purge) + rm -f /etc/default/stanchion + if [ -d /var/log/stanchion ]; then + rm -r /var/log/stanchion + fi + if [ -d /var/lib/stanchion ]; then + rm -r /var/lib/stanchion + fi + if [ -d /var/run/stanchion ]; then + rm -r /run/stanchion + fi + if [ -d /etc/stanchion ]; then + rm -r /etc/stanchion + fi + if [ -e /etc/init.d/stanchion ]; then + rm /etc/init.d/stanchion + fi + # Remove User & Group, killing any process owned by them + if getent passwd stanchion >/dev/null; then + pkill -u stanchion || true + deluser --quiet --system stanchion + fi + if getent group stanchion >/dev/null; then + delgroup --quiet --system --only-if-empty stanchion || true + fi + ;; + + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1\`" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/rel/pkg/deb/debian/rules b/rel/pkg/deb/debian/rules new file mode 100755 index 0000000..391e5ea --- /dev/null +++ b/rel/pkg/deb/debian/rules @@ -0,0 +1,55 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# modified for node_package by dizzyd@basho.com and jared@basho.com + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +ROOTDIR := debian/stanchion + +## Clear variables that may confound our build of sub-projects; also +## note that it is necessary to use overlay_vars relative to .. as +## the generate command EXECUTES in rel/ +build: + unset CC CFLAGS CPPFLAGS LDFLAGS CXX CXXFLAGS \ + && make rel-deb + touch build + +clean: + dh_clean + rm -f build + make clean + +## dh_shlibdeps was added to figure out the dependencies on shared libraries +## and will populate the ${shlibs:Depends} callout in the control file +install: LIBDIR := $(ROOTDIR)/usr/lib64/stanchion +install: build + dh_testdir + dh_testroot + dh_prep + dh_installdirs + dh_install + dh_installman + dh_fixperms + dh_shlibdeps + dh_systemd_enable + +# We have nothing to do by default. +binary-indep: install build-stamp +build-stamp: + +# Build architecture-dependent files here. +binary-arch: install + dh_strip -a + dh_compress -a + dh_installdeb + dh_gencontrol + dh_builddeb + +binary: binary-indep binary-arch diff --git a/rel/pkg/deb/debian/stanchion.manpages b/rel/pkg/deb/debian/stanchion.manpages new file mode 100644 index 0000000..e3db37c --- /dev/null +++ b/rel/pkg/deb/debian/stanchion.manpages @@ -0,0 +1 @@ +doc/man/man1/*.1.gz \ No newline at end of file diff --git a/rel/pkg/deb/debian/stanchion.service b/rel/pkg/deb/debian/stanchion.service new file mode 100644 index 0000000..5c8c824 --- /dev/null +++ b/rel/pkg/deb/debian/stanchion.service @@ -0,0 +1,14 @@ +[Unit] +Description=Serialization service for Riak CS + +[Service] +User=stanchion +ExecStart=/usr/sbin/stanchion foreground +ExecStop=/usr/sbin/stanchion stop +Type=simple +PIDFile=/run/stanchion/stanchion.pid +EnvironmentFile=-/etc/default/stanchion +RuntimeDirectory=stanchion + +[Install] +WantedBy=multi-user.target diff --git a/rel/pkg/deb/vars.config b/rel/pkg/deb/vars.config new file mode 100644 index 0000000..ce2a180 --- /dev/null +++ b/rel/pkg/deb/vars.config @@ -0,0 +1,46 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +{rel_vsn, "{{release_version}}"}. + +%% Platform-specific installation paths +{platform_bin_dir, "/usr/lib/stanchion/bin"}. +{platform_data_dir, "/var/lib/stanchion"}. +{platform_etc_dir, "/etc/stanchion"}. +{platform_base_dir, "/usr/lib/stanchion"}. +{platform_lib_dir, "/usr/lib/stanchion/lib"}. +{platform_log_dir, "/var/log/stanchion"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +%% +%% etc/app.config +%% +{stanchion_version, 030000}. +{stanchion_ip, "127.0.0.1"}. +{stanchion_port, 8085}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. + +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/run/stanchion"}. + +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. + +%% +%% cuttlefish +%% +{cuttlefish, "on"}. +{cuttlefish_conf, "stanchion.conf"}. + +%% lager +{console_log_default, file}. + +%% +%% etc/vm.args +%% +{node, "stanchion@127.0.0.1"}. diff --git a/rel/pkg/fbsdng/+MANIFEST b/rel/pkg/fbsdng/+MANIFEST new file mode 100644 index 0000000..2c4db06 --- /dev/null +++ b/rel/pkg/fbsdng/+MANIFEST @@ -0,0 +1,18 @@ +name: "stanchion" +origin: "databases" +comment: "Request Serialization Application" +licenses: ["Apache 2"] +licenselogic: "single" +arch: "freebsd:13:x86:64" +www: "tiot.jp" +maintainer: "packaging@tiot.jp" +users: ["stanchion"] +groups: ["stanchion"] +prefix: "/usr/local" +categories: ["databases"] +desc: "Request Serialization Application for Riak CS" +scripts: { + pre-install: "if ! pw groupshow stanchion 2>/dev/null; then pw groupadd stanchion; fi \n if ! pw usershow stanchion 2>/dev/null; then pw useradd stanchion -g stanchion -h - -d /usr/local/var/lib/stanchion -s /bin/sh -c \"Stanchion user\"; fi \n if [ ! -d /var/log/stanchion ]; then mkdir /var/log/stanchion && chown stanchion:stanchion /var/log/stanchion; fi" + post-install: "chown stanchion:stanchion /usr/local/etc/stanchion; (cd /usr/local/lib/stanchion; ln -s erts-* erts)" + pre-deinstall: "rm -f /usr/local/lib/stanchion/erts" +} diff --git a/rel/pkg/fbsdng/Makefile b/rel/pkg/fbsdng/Makefile new file mode 100644 index 0000000..2642923 --- /dev/null +++ b/rel/pkg/fbsdng/Makefile @@ -0,0 +1,107 @@ +export + +BUILDDIR = $(shell pwd) +BUILD_STAGE_DIR = $(BUILDDIR)/stanchion + +# Where we install things (based on vars.config) +# /usr/local based dirs +PMAN_DIR = $(BUILD_STAGE_DIR)/usr/local/man +PBIN_DIR = $(BUILD_STAGE_DIR)/usr/local/lib/stanchion/bin +PETC_DIR = $(BUILD_STAGE_DIR)/usr/local/etc/stanchion +PLIB_DIR = $(BUILD_STAGE_DIR)/usr/local/lib/stanchion +PUSRBIN_DIR = $(BUILD_STAGE_DIR)/usr/local/bin +# /var based dirs +PDATA_DIR = $(BUILD_STAGE_DIR)/usr/local/var/lib/stanchion +PLOG_DIR = $(BUILD_STAGE_DIR)/var/log/stanchion + +PKGNAME = stanchion-$(PKG_VERSION)-$(OSNAME)-$(ARCH).tbz + + +# Recursive assignment of ERTS version +# We will know this after building the rel +ERTS_PATH = $(shell ls $(BUILDDIR)/rel/stanchion | egrep -o "erts-.*") + +build: packing_list_files + @echo "Building package $(PKGNAME)" + + cd $(BUILD_STAGE_DIR) && \ + mkdir ../../../out/packages && \ + pkg create -m . -r . -o ../../../out/packages + + cd ../../out/packages && \ + for f in *.pkg; do \ + shasum -a 256 $${f} > $${f}.sha \ + ; done + +packing_list_files: $(BUILD_STAGE_DIR) + @mv ${BUILDDIR}/rel/pkg/fbsdng/+MANIFEST ${BUILD_STAGE_DIR} + sed -e "s/%ERTS_PATH%/${ERTS_PATH}/" < \ + ${BUILDDIR}/rel/pkg/fbsdng/rc.d > \ + ${BUILD_STAGE_DIR}/usr/local/etc/rc.d/stanchion + chmod -w ${BUILD_STAGE_DIR}/usr/local/etc/rc.d/stanchion + chmod +x ${BUILD_STAGE_DIR}/usr/local/etc/rc.d/stanchion + @cd $(BUILD_STAGE_DIR) && \ + echo "version: \"${PKG_VERSION}\"" >> +MANIFEST && \ + echo "files: {" >> +MANIFEST + + @echo "Copying Man pages to staging directory" + @cd $(BUILDDIR) && \ + if [ -d doc/man/man1 ]; then \ + mkdir -p $(PMAN_DIR) && \ + cp -R doc/man/man1 $(PMAN_DIR); fi + + @echo "Packaging /usr/local files" + @cd $(BUILD_STAGE_DIR) && \ + find usr -type f | while read file ; do \ + mode=$$(stat -f%p "$$file" | cut -c 3-) && \ + sum=$$(sha256 -q $$file) && \ + echo " \"/$$file\": { sum: \"$$sum\", perm: \"$$mode\", uname: \"root\", gname: \"wheel\" }," >> +MANIFEST; done && \ + sed -i .bak '$$s/,$$//' +MANIFEST && \ + rm -- +MANIFEST.bak && \ + echo " }" >> +MANIFEST + + @cd $(BUILD_STAGE_DIR) && \ + echo "directories: {" >> +MANIFEST && \ + echo " /usr/local/lib/stanchion: \"y\"," >> +MANIFEST && \ + echo " /usr/local/var/lib/stanchion: {uname: \"stanchion\", gname: \"stanchion\", perm: \"0700\" }," >> +MANIFEST && \ + echo " /usr/local/etc/stanchion: \"y\"" >> +MANIFEST && \ + echo " }" >> +MANIFEST + +# Copy the app rel directory to the staging directory to build our +# package structure and move the directories into the right place +# for the package, see the vars.config file for destination +# directories +$(BUILD_STAGE_DIR): buildrel + @echo "Copying rel directory to staging directory" + mkdir -p $@ + mkdir -p $(PBIN_DIR) $(PUSRBIN_DIR) + for f in stanchion-admin stanchion-debug; do \ + cp -R rel/stanchion/bin/$$f $(PUSRBIN_DIR); \ + done + cp -R rel/stanchion/usr/bin/stanchion $(PUSRBIN_DIR) + cp -R rel/stanchion/bin $(PLIB_DIR); + mkdir -p $(PETC_DIR) + cp -R rel/stanchion/etc/* $(PETC_DIR) + mkdir -p $(PLIB_DIR) + cp -R rel/stanchion/lib $(PLIB_DIR) + cp -R rel/stanchion/share $(PLIB_DIR) + cp -R rel/stanchion/erts-* $(PLIB_DIR) + (cd $(PLIB_DIR) && ln -s erts-* erts) + cp -R rel/stanchion/releases $(PLIB_DIR) + mkdir -p $(PDATA_DIR) +# cp -R rel/stanchion/data/* $(PDATA_DIR) || true # as of 3.0.0, there is nothing in data + mkdir -p ${BUILD_STAGE_DIR}/usr/local/etc/rc.d + +# Build the release we need to package +# * Ensure all binaries are executable +# * copy the vars.config over for build config +buildrel: + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/stanchion-$(PKG_ID) + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/stanchion-$(PKG_ID) rel-fbsdng + chmod 0755 rel/stanchion/bin/* rel/stanchion/erts-*/bin/* + +$(BUILDDIR): + mkdir -p $@ + +$(PKGERDIR)/pkgclean: + rm -rf $(BUILD_STAGE_DIR) $(BUILDDIR) diff --git a/rel/pkg/fbsdng/rc.d b/rel/pkg/fbsdng/rc.d new file mode 100644 index 0000000..b133245 --- /dev/null +++ b/rel/pkg/fbsdng/rc.d @@ -0,0 +1,19 @@ +#!/bin/sh + +# $FreeBSD$ +# +# PROVIDE: stanchion +# REQUIRE: LOGIN +# KEYWORD: shutdown + +. /etc/rc.subr + +name=stanchion +command=/usr/local/lib/stanchion/%ERTS_PATH%/bin/beam.smp +rcvar=stanchion_enable +start_cmd="/usr/local/bin/stanchion start" +stop_cmd="/usr/local/bin/stanchion stop" +pidfile="/var/run/stanchion/stanchion.pid" + +load_rc_config $name +run_rc_command "$1" diff --git a/rel/pkg/fbsdng/vars.config b/rel/pkg/fbsdng/vars.config new file mode 100644 index 0000000..2061b93 --- /dev/null +++ b/rel/pkg/fbsdng/vars.config @@ -0,0 +1,46 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +{rel_vsn, "{{release_version}}"}. + +%% Platform-specific installation paths +{platform_bin_dir, "/usr/local/lib/stanchion/bin"}. +{platform_data_dir, "/usr/local/var/lib/stanchion"}. +{platform_etc_dir, "/usr/local/etc/stanchion"}. +{platform_base_dir, "/usr/local/lib/stanchion"}. +{platform_lib_dir, "/usr/local/lib/stanchion/lib"}. +{platform_log_dir, "/var/log/stanchion"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +%% +%% etc/app.config +%% +{stanchion_version, 030000}. +{stanchion_ip, "127.0.0.1"}. +{stanchion_port, 8085}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. + +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/var/run/stanchion"}. + +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. + +%% +%% cuttlefish +%% +{cuttlefish, "on"}. +{cuttlefish_conf, "stanchion.conf"}. + +%% lager +{console_log_default, file}. + +%% +%% etc/vm.args +%% +{node, "stanchion@127.0.0.1"}. diff --git a/rel/pkg/osx/Makefile b/rel/pkg/osx/Makefile new file mode 100644 index 0000000..a97eb4b --- /dev/null +++ b/rel/pkg/osx/Makefile @@ -0,0 +1,13 @@ +BUILDDIR = $(shell pwd) +BUILD_STAGE_DIR = $(BUILDDIR)/stanchion +PKGNAME = stanchion-$(PKG_ID) + +default: buildrel + mkdir ../packages + tar -czf ../packages/$(PKGNAME).tar.gz -C rel stanchion && \ + (cd ../packages && shasum -a 256 $(PKGNAME).tar.gz > $(PKGNAME).tar.gz.sha) + +buildrel: + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/$(PKGNAME) + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/$(PKGNAME) rel-osx + chmod 0755 rel/stanchion/bin/* rel/stanchion/erts-*/bin/* diff --git a/rel/pkg/osx/vars.config b/rel/pkg/osx/vars.config new file mode 120000 index 0000000..0a676b8 --- /dev/null +++ b/rel/pkg/osx/vars.config @@ -0,0 +1 @@ +../../vars.config \ No newline at end of file diff --git a/rel/pkg/rpm/Makefile b/rel/pkg/rpm/Makefile new file mode 100644 index 0000000..da19ab2 --- /dev/null +++ b/rel/pkg/rpm/Makefile @@ -0,0 +1,24 @@ + +PWD = $(shell pwd) + + +# No hyphens are allowed in the _version field in RPM +PKG_VERSION_NO_H ?= $(shell echo $(PKG_VERSION) | tr - .) + +default: + rpmbuild --define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}$(DISTRO).%%{ARCH}.rpm" \ + --define '_topdir $(BASE_DIR)/rel/pkg/out/' \ + --define '_sourcedir $(BASE_DIR)/rel/pkg/out/' \ + --define '_specdir $(BASE_DIR)/rel/pkg/out/' \ + --define '_rpmdir $(BASE_DIR)/rel/pkg/out/packages' \ + --define '_srcrpmdir $(BASE_DIR)/rel/pkg/out/packages' \ + --define "_revision $(PKG_VERSION)" \ + --define "_version $(PKG_VERSION_NO_H)" \ + --define "_release $(PKG_BUILD)" \ + --define "_tarname $(PKG_ID).tar.gz" \ + --define "_tarname_base $(PKG_ID)" \ + -ba $(BASE_DIR)/rel/pkg/rpm/specfile + cd $(BASE_DIR)/rel/pkg/out/packages && \ + for rpmfile in *.rpm; do \ + sha256sum $${rpmfile} > $${rpmfile}.sha \ + ; done diff --git a/rel/pkg/rpm/specfile b/rel/pkg/rpm/specfile new file mode 100644 index 0000000..c0dda3d --- /dev/null +++ b/rel/pkg/rpm/specfile @@ -0,0 +1,160 @@ +## ------------------------------------------------------------------- +## +## Copyright (c) 2014 Basho Technologies, Inc.; 2021 TI Tokyo +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## ------------------------------------------------------------------- + +Name: stanchion +Version: %{_version} +Release: %{_release}%{?dist} +License: "Apache 2.0" +Group: Development/Libraries +Source: %{_tarname} +URL: "https://tiot.jp" +Vendor: "TI Tokyo" +Packager: "Stanchion Package Maintainer" <"andriy.zavada@tiot.jp"> +BuildRoot: %{_tmppath}/%{name}-%{_revision}-%{release}-root +Summary: "Request serialization service for Riak CS" +BuildRequires: systemd +Requires: logrotate +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd + +%description +"Stanchion is a request serialization service for Riak CS" + +%define debug_package %{nil} +%define __prelink_undo_cmd /bin/cat prelink library + +%define platform_etc_dir %{_sysconfdir}/stanchion +%define platform_base_dir %{_libdir}/stanchion +%define platform_bin_dir %{platform_base_dir}/bin +%define platform_lib_dir %{platform_base_dir}/lib +%define platform_log_dir %{_localstatedir}/log/stanchion + +%prep +%setup -q -n %{_tarname_base} -c %{_tarname_base} + +# Setup vars.config like other platforms, but do it inside of spec file +cat > rpm.vars.config </dev/null 2>&1; then + groupadd -r stanchion +fi + +if getent passwd stanchion >/dev/null 2>&1; then + usermod -d %{_localstatedir}/lib/stanchion stanchion || true +else + useradd -r -g stanchion \ + --home %{_localstatedir}/lib/stanchion \ + --comment "Stanchion User" \ + --shell /bin/bash \ + stanchion +fi + +%post +# For distros with SELinux (RHEL/Fedora) +if [ `which selinuxenabled > /dev/null 2>&1` ] ; then + # Fixup perms for SELinux (if it is enabled) + selinuxenabled && find %{_localstatedir}/lib/stanchion -name "*.so" -exec chcon -t textrel_shlib_t {} \; +fi + +%systemd_post %{name}.service + +%preun +%systemd_preun %{name}.service + +%postun +%systemd_postun %{name}.service +rm -f %{_sysconfdir}/stanchion/stanchion.conf.new +rm -rf %{_localstatedir}/lib/stanchion +rm -rf %{_localstatedir}/log/stanchion + +%files +%defattr(-,root,root) +%{_libdir}/* +%{_unitdir}/%{name}.service +%{_sbindir}/* + +%defattr(-,stanchion,stanchion) +%{_sysconfdir}/stanchion +%{_localstatedir}/lib/stanchion +%{_localstatedir}/log/stanchion + +%config(noreplace) %{_sysconfdir}/stanchion/* + +%clean +rm -rf %{buildroot} diff --git a/rel/pkg/rpm/stanchion.service b/rel/pkg/rpm/stanchion.service new file mode 100644 index 0000000..710d15e --- /dev/null +++ b/rel/pkg/rpm/stanchion.service @@ -0,0 +1,15 @@ +[Unit] +Description=Serialization service for Riak CS + +[Service] +User=stanchion +ExecStart=/usr/sbin/stanchion foreground +ExecStop=/usr/sbin/stanchion stop +Type=simple + +PIDFile=/run/stanchion/stanchion.pid +EnvironmentFile=-/etc/default/stanchion +RuntimeDirectory=stanchion + +[Install] +WantedBy=multi-user.target diff --git a/rel/pkg/rpm/vars.config b/rel/pkg/rpm/vars.config new file mode 100644 index 0000000..b7cb49f --- /dev/null +++ b/rel/pkg/rpm/vars.config @@ -0,0 +1,49 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +{rel_vsn, "{{release_version}}"}. + +%% Platform-specific installation paths +{platform_bin_dir, "/usr/lib64/stanchion/bin"}. +{platform_data_dir, "/var/lib/stanchion"}. +{platform_etc_dir, "/etc/stanchion"}. +{platform_base_dir, "/usr/lib64/stanchion"}. +{platform_lib_dir, "/usr/lib64/stanchion/lib"}. +{platform_log_dir, "/var/log/stanchion"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +%% +%% etc/app.config +%% +{stanchion_version, 030000}. +{stanchion_ip, "127.0.0.1"}. +{stanchion_port, 8085}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. + +%% +%% etc/vm.args +%% +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/run/stanchion"}. + +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. + +%% +%% cuttlefish +%% +{cuttlefish, "on"}. +{cuttlefish_conf, "stanchion.conf"}. + +%% lager +{console_log_default, file}. + +%% +%% etc/vm.args +%% +{node, "stanchion@127.0.0.1"}. diff --git a/rel/rebar.config b/rel/rebar.config deleted file mode 100644 index 09b7ba7..0000000 --- a/rel/rebar.config +++ /dev/null @@ -1,3 +0,0 @@ -{plugin_dir, "../deps/cuttlefish/src"}. -{plugins, [cuttlefish_rebar_plugin]}. -{cuttlefish_filename, "stanchion.conf"}. diff --git a/rel/reltool.config b/rel/reltool.config deleted file mode 100644 index 20197c5..0000000 --- a/rel/reltool.config +++ /dev/null @@ -1,75 +0,0 @@ -%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et -{sys, [ - {lib_dirs, ["../deps", "../apps"]}, - {rel, "stanchion", "2.1.0", - [ - kernel, - stdlib, - sasl, - public_key, - ssl, - os_mon, - crypto, - runtime_tools, - mochiweb, - webmachine, - lager, - lager_syslog, - exometer_core, - stanchion - ]}, - {rel, "start_clean", "", - [ - kernel, - stdlib - ]}, - {boot_rel, "stanchion"}, - {profile, embedded}, - {excl_sys_filters, ["^bin/.*", - "^erts.*/bin/(dialyzer|typer)"]}, - {excl_archive_filters, [".*"]}, - {app, sasl, [{incl_cond, include}]}, - {app, cuttlefish, [{incl_cond, include}]}, - {app, lager, [{incl_cond, include}]}, - {app, stanchion, [{incl_cond, include}]} - ]}. - - -{target_dir, "stanchion"}. - -{overlay_vars, "vars.config"}. - -{overlay, [ - %% Scan for scripts in included apps - %% Setup basic dirs - {mkdir, "log"}, - {mkdir, "data/stanchion"}, - - %% Copy base files for starting and interacting w/ node - {copy, "../deps/node_package/priv/base/erl", - "{{erts_vsn}}/bin/erl"}, - {copy, "../deps/cuttlefish/cuttlefish", - "{{erts_vsn}}/bin/cuttlefish"}, - {copy, "../deps/node_package/priv/base/nodetool", - "{{erts_vsn}}/bin/nodetool"}, - {template, "../deps/node_package/priv/base/runner", - "bin/stanchion"}, - {template, "../deps/node_package/priv/base/env.sh", - "lib/env.sh"}, - - %% Copy config files - %% Cuttlefish Schema Files have a priority order. - %% Anything in a file prefixed with 00- will override - %% anything in a file with a higher numbered prefix. - {template, "files/stanchion.schema", "lib/00-stanchion.schema"}, - {template, "../deps/cuttlefish/priv/erlang_vm.schema", "lib/11-erlang_vm.schema"}, - - {template, "files/advanced.config", "etc/advanced.config"}, - {template, "files/stanchion-admin", "bin/stanchion-admin"}, - - %% Copy ssl certs - {template, "files/cert.pem", "etc/cert.pem"}, - {template, "files/key.pem", "etc/key.pem"} - - ]}. diff --git a/rel/vars.config b/rel/vars.config index cae78d4..bb1fe25 100644 --- a/rel/vars.config +++ b/rel/vars.config @@ -1,23 +1,27 @@ -%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et +%% -*- erlang -*- + +{rel_vsn, "{{release_version}}"}. %% Platform-specific installation paths -{platform_bin_dir, "./bin"}. -{platform_data_dir, "./data"}. -{platform_etc_dir, "./etc"}. -{platform_lib_dir, "./lib"}. -{platform_log_dir, "./log"}. +{platform_base_dir, "${STANCHION_PATH:-$RELEASE_ROOT_DIR}"}. +{platform_bin_dir, "./bin"}. +{platform_data_dir, "./data"}. +{platform_etc_dir, "./etc"}. +{platform_lib_dir, "./lib"}. +{platform_log_dir, "./log"}. +{platform_gen_dir, "."}. +{platform_patch_dir, "./patches"}. %% %% etc/app.config %% +{stanchion_version, 030000}. {stanchion_ip, "127.0.0.1"}. {stanchion_port, 8085}. {riak_ip, "127.0.0.1"}. {riak_pb_port, 8087}. {auth_bypass, false}. {admin_key, "admin-key"}. -{admin_secret, "admin-secret"}. %% %% etc/vm.args @@ -25,23 +29,9 @@ {node, "stanchion@127.0.0.1"}. {crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. -%% -%% bin/stanchion -%% -{data_dir, "{{target_dir}}/data"}. -{runner_script_dir, "\`cd \\`dirname $0\\` && /bin/pwd\`"}. -{runner_base_dir, "{{runner_script_dir}}/.."}. -{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. -{runner_log_dir, "$RUNNER_BASE_DIR/log"}. -{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. -{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. -{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. -{runner_user, ""}. -{runner_wait_process, "stanchion_server_sup"}. +%% relocatable releases won't call the launcher script, because +%% launcher script requires a stanchion user to exist. +%%{pid_dir, "$PLATFORM_BASE_DIR/var/run/stanchion"}. %% lager {console_log_default, file}. - -%% cuttlefish -{cuttlefish, "on"}. -{cuttlefish_conf, "stanchion.conf"}. diff --git a/riak_test/bin/rtdev-build-releases.sh b/riak_test/bin/rtdev-build-releases.sh deleted file mode 100755 index 973dc07..0000000 --- a/riak_test/bin/rtdev-build-releases.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -# You need to use this script once to build a set of devrels for prior -# releases of Riak (for mixed version / upgrade testing). You should -# create a directory and then run this script from within that directory. -# I have ~/test-releases that I created once, and then re-use for testing. -# -# See rtdev-setup-releases.sh as an example of setting up mixed version layout -# for testing. - -# Different versions of Riak were released using different Erlang versions, -# make sure to build with the appropriate version. - -# This is based on my usage of having multiple Erlang versions in different -# directories. If using kerl or whatever, modify to use kerl's activate logic. -# Or, alternatively, just substitute the paths to the kerl install paths as -# that should work too. - -R15B01=${R15B01:-$HOME/erlang/R15B01-64} -R16B02=${R16B02:-$HOME/erlang/R16B02-64} - -checkbuild() -{ - ERLROOT=$1 - - if [ ! -d $ERLROOT ]; then - echo -n "$ERLROOT cannot be found, install kerl? [y|N]: " - read ans - if [[ $ans == n || $ans == N ]]; then - exit 1 - fi - fi -} - -kerl() -{ - RELEASE=$1 - BUILDNAME=$2 - - if [ ! -x kerl ]; then - curl -O https://raw.github.com/spawngrid/kerl/master/kerl; chmod a+x kerl - fi - - ./kerl build $RELEASE $BUILDNAME - ./kerl install $BUILDNAME $HOME/$BUILDNAME -} - -build() -{ - SRCDIR=$1 - ERLROOT=$2 - - if [ ! -d $ERLROOT ]; then - BUILDNAME=`basename $ERLROOT` - RELEASE=`echo $BUILDNAME | awk -F- '{ print $2 }'` - kerl $RELEASE $BUILDNAME - fi - - echo - echo "Building $SRCDIR" - cd $SRCDIR - - RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ - C_INCLUDE_PATH=$ERLROOT/usr/include \ - LD_LIBRARY_PATH=$ERLROOT/usr/lib" - echo $RUN - $RUN make && $RUN make devrel - cd .. -} - -download() -{ - URI=$1 - FILENAME=`echo $URI | awk -F/ '{ print $8 }'` - if [ ! -f $FILENAME ]; then - wget $URI - fi -} - - -checkbuild $R15B01 -checkbuild $R16B02 - -download http://s3.amazonaws.com/downloads.basho.com/stanchion/1.5/1.5.0/stanchion-1.5.0.tar.gz - -tar -xf stanchion-1.5.0.tar.gz -build "stanchion-1.5.0" $R15B01 diff --git a/riak_test/bin/rtdev-current.sh b/riak_test/bin/rtdev-current.sh deleted file mode 100755 index 38ce485..0000000 --- a/riak_test/bin/rtdev-current.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# bail out if things go south -set -e - -: ${RTSTANCHION_DEST_DIR:="$HOME/rt/stanchion"} - -cwd=$(pwd) -cd $RTSTANCHION_DEST_DIR -git reset HEAD --hard -git clean -fd -rm -rf $RTSTANCHION_DEST_DIR/current -mkdir $RTSTANCHION_DEST_DIR/current -cd $cwd -cp -a dev $RTSTANCHION_DEST_DIR/current -cd $RTSTANCHION_DEST_DIR -git add --all . -git commit -a -m "riak_test init" --amend > /dev/null - diff --git a/riak_test/bin/rtdev-setup-releases.sh b/riak_test/bin/rtdev-setup-releases.sh deleted file mode 100755 index 9475742..0000000 --- a/riak_test/bin/rtdev-setup-releases.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# bail out if things go south -set -e - -# Creates a mixed-version directory structure for running riak_test -# using rtdev-mixed.config settings. Should be run inside a directory -# that contains devrels for prior Riak CS releases. Easy way to create this -# is to use the rtdev-build-releases.sh script - -: ${RTSTANCHION_DEST_DIR:="$HOME/rt/stanchion"} - -rm -rf $RTSTANCHION_DEST_DIR -mkdir $RTSTANCHION_DEST_DIR - -count=$(ls */dev 2> /dev/null | wc -l) -if [ "$count" -ne "0" ] -then - for rel in */dev; do - vsn=$(dirname "$rel") - echo " - Initializing $RTSTANCHION_DEST_DIR/$vsn" - mkdir -p "$RTSTANCHION_DEST_DIR/$vsn" - cp -p -P -R "$rel" "$RTSTANCHION_DEST_DIR/$vsn" - done -else - # This is useful when only testing with 'current' - # The repo still needs to be initialized for current - # and we don't want to bomb out if */dev doesn't exist - touch $RTSTANCHION_DEST_DIR/.current_init - echo "No devdirs found. Not copying any releases." -fi - -cd $RTSTANCHION_DEST_DIR -git init - -## Some versions of git and/or OS require these fields -git config user.name "Riak Test" -git config user.email "dev@basho.com" - -git add . -git commit -a -m "riak_test init" > /dev/null -echo " - Successfully completed initial git commit of $RTSTANCHION_DEST_DIR" diff --git a/src/stanchion_app.erl b/src/stanchion_app.erl deleted file mode 100644 index 621441e..0000000 --- a/src/stanchion_app.erl +++ /dev/null @@ -1,54 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Callbacks for the stanchion application. - --module(stanchion_app). - --behaviour(application). - -%% application API --export([start/2, - stop/1]). - - --type start_type() :: normal | {takeover, node()} | {failover, node()}. --type start_args() :: term(). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc application start callback for stanchion. --spec start(start_type(), start_args()) -> {ok, pid()} | - {error, term()}. -start(_Type, _StartArgs) -> - case stanchion_utils:riak_connection() of - {ok, _} -> - stanchion_sup:start_link(); - {error, Reason} -> - _ = lager:error("Couldn't connect to Riak: ~p", [Reason]), - {error, Reason} - end. - -%% @doc application stop callback for stanchion. --spec stop(term()) -> ok. -stop(_State) -> - ok. diff --git a/src/stanchion_manifest_resolution.erl b/src/stanchion_manifest_resolution.erl deleted file mode 100644 index 91df771..0000000 --- a/src/stanchion_manifest_resolution.erl +++ /dev/null @@ -1,162 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Module to resolve siblings with manifest records - --module(stanchion_manifest_resolution). - --include("riak_moss.hrl"). - -%% export Public API --export([resolve/1]). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc Take a list of siblings -%% and resolve them to a single -%% value. In this case, siblings -%% and values are dictionaries whose -%% keys are UUIDs and whose values -%% are manifests. --spec resolve(list()) -> term(). -resolve(Siblings) -> - lists:foldl(fun resolve_dicts/2, orddict:new(), Siblings). - -%% ==================================================================== -%% Internal functions -%% ==================================================================== - -%% @doc Take two dictionaries -%% of manifests and resolve them. -%% @private --spec resolve_dicts(term(), term()) -> term(). -resolve_dicts(A, B) -> - orddict:merge(fun resolve_manifests/3, A, B). - -%% @doc Take two manifests with -%% the same UUID and resolve them -%% @private --spec resolve_manifests(term(), term(), term()) -> term(). -resolve_manifests(_Key, A, B) -> - AState = state_to_stage_number(A?MANIFEST.state), - BState = state_to_stage_number(B?MANIFEST.state), - resolve_manifests(AState, BState, A, B). - -state_to_stage_number(writing) -> 10; -state_to_stage_number(active) -> 20; -state_to_stage_number(pending_delete) -> 30; -state_to_stage_number(scheduled_delete) -> 40. - -%% @doc Return a new, resolved manifest. -%% The first two args are the state that -%% manifest A and B are in, respectively. -%% The third and fourth args, A, B, are the -%% manifests themselves. -%% @private --spec resolve_manifests(integer(), integer(), term(), term()) -> term(). - -resolve_manifests(StageA, StageB, A, _B) when StageA > StageB -> - A; -resolve_manifests(StageA, StageB, _A, B) when StageB > StageA -> - B; -resolve_manifests(StageX, StageX, A, A) -> - A; -resolve_manifests(_, _, - ?MANIFEST{state = writing} = A, - ?MANIFEST{state = writing} = B) -> - WriteBlocksRemaining = resolve_written_blocks(A, B), - LastBlockWrittenTime = resolve_last_written_time(A, B), - A?MANIFEST{write_blocks_remaining=WriteBlocksRemaining, last_block_written_time=LastBlockWrittenTime}; - -%% Check for and handle differing ACLs, but otherwise purposely throw -%% a function clause exception if the manifests aren't equivalent -resolve_manifests(_, _, - A1=?MANIFEST{acl=A1Acl}, - A2=?MANIFEST{acl=A2Acl}) when A1Acl =/= A2Acl -> - case A1Acl?ACL.creation_time >= A2Acl?ACL.creation_time of - true -> - A1; - false -> - A2 - end; - -resolve_manifests(_, _, - ?MANIFEST{state = pending_delete} = A, - ?MANIFEST{state = pending_delete} = B) -> - BlocksLeftToDelete = resolve_deleted_blocks(A, B), - LastDeletedTime = resolve_last_deleted_time(A, B), - A?MANIFEST{delete_blocks_remaining=BlocksLeftToDelete, - last_block_deleted_time=LastDeletedTime}; - -resolve_manifests(_, _, - ?MANIFEST{state = scheduled_delete} = A, - ?MANIFEST{state = scheduled_delete} = B) -> - BlocksLeftToDelete = resolve_deleted_blocks(A, B), - LastDeletedTime = resolve_last_deleted_time(A, B), - A?MANIFEST{delete_blocks_remaining=BlocksLeftToDelete, - last_block_deleted_time=LastDeletedTime}. - -resolve_written_blocks(A, B) -> - AWritten = A?MANIFEST.write_blocks_remaining, - BWritten = B?MANIFEST.write_blocks_remaining, - ordsets:intersection(AWritten, BWritten). - -resolve_deleted_blocks(A, B) -> - ADeleted = A?MANIFEST.delete_blocks_remaining, - BDeleted = B?MANIFEST.delete_blocks_remaining, - safe_intersection(ADeleted, BDeleted). - -%% NOTE: -%% There was a bit of a gaff -%% and delete_blocks_remaining -%% was not set to an ordset -%% when the state was set to -%% pending_delete, so we have -%% to account for it being -%% `undefined` -safe_intersection(undefined, undefined) -> - %% if these are both - %% undefined, then - %% neither have ever had - %% delete_blocks_remaining set - %% as something meaningful, - %% so don't just change it - %% to they empty set. - undefined; -safe_intersection(A, undefined) -> - safe_intersection(A, []); -safe_intersection(undefined, B) -> - safe_intersection([], B); -safe_intersection(A, B) -> - ordsets:intersection(A, B). - -resolve_last_written_time(A, B) -> - ALastWritten = A?MANIFEST.last_block_written_time, - BLastWritten = B?MANIFEST.last_block_written_time, - latest_date(ALastWritten, BLastWritten). - -resolve_last_deleted_time(A, B) -> - ALastDeleted = A?MANIFEST.last_block_deleted_time, - BLastDeleted = B?MANIFEST.last_block_deleted_time, - latest_date(ALastDeleted, BLastDeleted). - -latest_date(A, B) -> erlang:max(A, B). diff --git a/src/stanchion_manifest_utils.erl b/src/stanchion_manifest_utils.erl deleted file mode 100644 index 359569e..0000000 --- a/src/stanchion_manifest_utils.erl +++ /dev/null @@ -1,213 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Module for choosing and manipulating lists (well, orddict) of manifests - --module(stanchion_manifest_utils). - --include("riak_moss.hrl"). --ifdef(TEST). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --endif. - -%% export Public API --export([new_dict/2, - active_manifest/1, - active_and_writing_manifests/1, - overwritten_UUIDs/1, - mark_pending_delete/2, - mark_scheduled_delete/2, - upgrade_wrapped_manifests/1, - upgrade_manifest/1]). - -%%%=================================================================== -%%% API -%%%=================================================================== - -%% @doc Return a new orddict of manifest (only -%% one in this case). Used when storing something -%% in Riak when the previous GET returned notfound, -%% so we're (maybe) creating a new object. --spec new_dict(binary(), lfs_manifest()) -> orddict:orddict(). -new_dict(UUID, Manifest) -> - orddict:store(UUID, Manifest, orddict:new()). - -%% @doc Return the current active manifest -%% from an orddict of manifests. --spec active_manifest(orddict:orddict()) -> {ok, lfs_manifest()} | {error, no_active_manifest}. -active_manifest(Dict) -> - case lists:foldl(fun most_recent_active_manifest/2, no_active_manifest, orddict_values(Dict)) of - no_active_manifest -> - {error, no_active_manifest}; - Manifest -> - {ok, Manifest} - end. - -%% @doc Return a list of all manifests in the -%% `active' or `writing' state --spec active_and_writing_manifests(orddict:orddict()) -> [lfs_manifest()]. -active_and_writing_manifests(Dict) -> - orddict:to_list(filter_manifests_by_state(Dict, [active, writing])). - -%% @doc Extract all manifests that are not "the most active" -%% and not actively writing (within the leeway period). --spec overwritten_UUIDs(orddict:orddict()) -> term(). -overwritten_UUIDs(Dict) -> - case active_manifest(Dict) of - {error, no_active_manifest} -> - FoldFun = - fun ({_, ?MANIFEST{state=State}}, Acc) when State =:= writing -> - Acc; - ({UUID, _}, Acc) -> - [UUID | Acc] - end; - {ok, Active} -> - FoldFun = - fun ({UUID, Elem}, Acc) -> - case Elem of - Active -> - Acc; - Elem=?MANIFEST{state=active} -> - [UUID | Acc]; - Elem=?MANIFEST{state=writing} -> - [UUID | Acc]; - _ -> - Acc - end - end - end, - lists:foldl(FoldFun, [], orddict:to_list(Dict)). - -%% @doc Return `Dict' with the manifests in -%% `UUIDsToMark' with their state changed to -%% `pending_delete' --spec mark_pending_delete(orddict:orddict(), list(binary())) -> - orddict:orddict(). -mark_pending_delete(Dict, UUIDsToMark) -> - MapFun = fun(K, V) -> - case lists:member(K, UUIDsToMark) of - true -> - V?MANIFEST{state=pending_delete, - delete_marked_time=os:timestamp()}; - false -> - V - end - end, - orddict:map(MapFun, Dict). - -%% @doc Return `Dict' with the manifests in -%% `UUIDsToMark' with their state changed to -%% `scheduled_delete' --spec mark_scheduled_delete(orddict:orddict(), list(binary())) -> - orddict:orddict(). -mark_scheduled_delete(Dict, UUIDsToMark) -> - MapFun = fun(K, V) -> - case lists:member(K, UUIDsToMark) of - true -> - V?MANIFEST{state=scheduled_delete, - scheduled_delete_time=os:timestamp()}; - false -> - V - end - end, - orddict:map(MapFun, Dict). - --spec upgrade_wrapped_manifests([orddict:orddict()]) -> [orddict:orddict()]. -upgrade_wrapped_manifests(ListofOrdDicts) -> - DictMapFun = fun (_Key, Value) -> upgrade_manifest(Value) end, - MapFun = fun (Value) -> orddict:map(DictMapFun, Value) end, - lists:map(MapFun, ListofOrdDicts). - -%% @doc Upgrade the manifest to the most recent -%% version of the manifest record. This is so that -%% _most_ of the codebase only has to deal with -%% the most recent version of the record. --spec upgrade_manifest(lfs_manifest() | #lfs_manifest_v2{}) -> lfs_manifest(). -upgrade_manifest(#lfs_manifest_v2{block_size=BlockSize, - bkey=Bkey, - metadata=Metadata, - created=Created, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - state=State, - write_start_time=WriteStartTime, - last_block_written_time=LastBlockWrittenTime, - write_blocks_remaining=WriteBlocksRemaining, - delete_marked_time=DeleteMarkedTime, - last_block_deleted_time=LastBlockDeletedTime, - delete_blocks_remaining=DeleteBlocksRemaining, - acl=Acl, - props=Properties, - cluster_id=ClusterID}) -> - - ?MANIFEST{block_size=BlockSize, - bkey=Bkey, - metadata=Metadata, - created=Created, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - state=State, - write_start_time=WriteStartTime, - last_block_written_time=LastBlockWrittenTime, - write_blocks_remaining=WriteBlocksRemaining, - delete_marked_time=DeleteMarkedTime, - last_block_deleted_time=LastBlockDeletedTime, - delete_blocks_remaining=DeleteBlocksRemaining, - acl=Acl, - props=Properties, - cluster_id=ClusterID}; - -upgrade_manifest(?MANIFEST{}=M) -> - M. - -%%%=================================================================== -%%% Internal functions -%%%=================================================================== - -%% @doc Filter an orddict manifests and accept only manifests whose -%% current state is specified in the `AcceptedStates' list. --spec filter_manifests_by_state(orddict:orddict(), [atom()]) -> orddict:orddict(). -filter_manifests_by_state(Dict, AcceptedStates) -> - AcceptManifest = - fun (_, ?MANIFEST{state=State}) -> - lists:member(State, AcceptedStates) - end, - orddict:filter(AcceptManifest, Dict). - -orddict_values(OrdDict) -> - [V || {_K, V} <- orddict:to_list(OrdDict)]. - -%% NOTE: This is a foldl function, initial acc = no_active_manifest -most_recent_active_manifest(Manifest=?MANIFEST{state=active}, no_active_manifest) -> - Manifest; -most_recent_active_manifest(_Manfest, no_active_manifest) -> - no_active_manifest; -most_recent_active_manifest(Man1=?MANIFEST{state=active}, Man2=?MANIFEST{state=active}) -> - case Man1?MANIFEST.write_start_time > Man2?MANIFEST.write_start_time of - true -> Man1; - false -> Man2 - end; -most_recent_active_manifest(Man1=?MANIFEST{state=active}, _Man2) -> Man1; -most_recent_active_manifest(_Man1, Man2=?MANIFEST{state=active}) -> Man2. diff --git a/src/stanchion_multipart.erl b/src/stanchion_multipart.erl deleted file mode 100644 index ed5ffc2..0000000 --- a/src/stanchion_multipart.erl +++ /dev/null @@ -1,294 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(stanchion_multipart). - --export([check_no_multipart_uploads/2]). - --include("stanchion.hrl"). - --type cluster_id() :: undefined | binary(). % Type still in flux. - --type cs_uuid() :: binary(). - --record(lfs_manifest_v2, { - version=2 :: integer(), - block_size :: integer(), - bkey :: {binary(), binary()}, - metadata :: orddict:orddict(), - created=riak_moss_wm_utils:iso_8601_datetime(), - uuid :: cs_uuid(), - content_length :: non_neg_integer(), - content_type :: binary(), - content_md5 :: term(), - state=undefined :: undefined | writing | active | - pending_delete | scheduled_delete | deleted, - write_start_time :: term(), %% immutable - last_block_written_time :: term(), - write_blocks_remaining :: ordsets:ordset(integer()), - delete_marked_time :: term(), - last_block_deleted_time :: term(), - delete_blocks_remaining :: ordsets:ordset(integer()), - acl :: acl(), - props = [] :: proplists:proplist(), - cluster_id :: cluster_id() - }). - --record(lfs_manifest_v3, { - %% "global" properties - %% ----------------------------------------------------------------- - - %% this isn't as important anymore - %% since we're naming the record - %% to include the version number, - %% but I figured it's worth keeping - %% in case we change serialization - %% formats in the future. - version=3 :: integer(), - - %% the block_size setting when this manifest - %% was written. Needed if the user - %% ever changes the block size after writing - %% data - block_size :: integer(), - - %% identifying properties - %% ----------------------------------------------------------------- - bkey :: {binary(), binary()}, - - %% user metadata that would normally - %% be placed on the riak_object. We avoid - %% putting it on the riak_object so that - %% we can use that metadata ourselves - metadata :: orddict:orddict(), - - %% the date the manifest was created. - %% not sure if we need both this and - %% write_start_time. My thought was that - %% write_start_time would have millisecond - %% resolution, but I suppose there's no - %% reason we can't change created - %% to have millisecond as well. - created=riak_moss_wm_utils:iso_8601_datetime(), - uuid :: cs_uuid(), - - %% content properties - %% ----------------------------------------------------------------- - content_length :: non_neg_integer(), - content_type :: binary(), - content_md5 :: term(), - - %% state properties - %% ----------------------------------------------------------------- - state=undefined :: undefined | writing | active | - pending_delete | scheduled_delete | deleted, - - %% writing/active state - %% ----------------------------------------------------------------- - write_start_time :: term(), %% immutable - - %% used for two purposes - %% 1. to mark when a file has finished uploading - %% 2. to decide if a write crashed before completing - %% and needs to be garbage collected - last_block_written_time :: term(), - - %% a shrink-only (during resolution) - %% set to denote which blocks still - %% need to be written. We use a shrinking - %% (rather than growing) set to that the - %% set is empty after the write has completed, - %% which should be most of the lifespan on disk - write_blocks_remaining :: ordsets:ordset(integer()), - - %% pending_delete/deleted state - %% ----------------------------------------------------------------- - %% set to the current time - %% when a manifest is marked as deleted - %% and enters the pending_delete state - delete_marked_time :: term(), - - %% the timestamp serves a similar - %% purpose to last_block_written_time, - %% in that it's used for figuring out - %% when delete processes have died - %% and garbage collection needs to - %% pick up where they left off. - last_block_deleted_time :: term(), - - %% a shrink-only (during resolution) - %% set to denote which blocks - %% still need to be deleted. - %% See write_blocks_remaining for - %% an explanation of why we chose - %% a shrinking set - delete_blocks_remaining :: ordsets:ordset(integer()), - - %% the time the manifest was put - %% into the scheduled_delete - %% state - scheduled_delete_time :: term(), - - %% The ACL for the version of the object represented - %% by this manifest. - acl :: acl(), - - %% There are a couple of cases where we want to add record - %% member'ish data without adding new members to the record, - %% e.g. - %% 1. Data for which the common value is 'undefined' or not - %% used/set for this particular manifest - %% 2. Cases where we do want to change the structure of the - %% record but don't want to go through the full code - %% refactoring and backward-compatibility tap dance - %% until sometime later. - props = [] :: proplists:proplist(), - - %% cluster_id: A couple of uses, both short- and longer-term - %% possibilities: - %% - %% 1. We don't have a good story in early 2012 for how to - %% build a stable 2,000 node Riak cluster. If MOSS can - %% talk to multiple Riak clusters, then each individual - %% cluster can be a size that we're comfortable - %% supporting. - %% - %% 2. We may soon have Riak EE's replication have full - %% plumbing to make it feasible to forward arbitrary - %% traffic between clusters. Then if a slave cluster is - %% missing a data block, and read-repair cannot - %% automagically fix the 'not_found' problem, then perhaps - %% forwarding a get request to the source Riak cluster can - %% fetch us the missing data. - cluster_id :: cluster_id() - }). --type lfs_manifest() :: #lfs_manifest_v3{}. --define(MANIFEST, #lfs_manifest_v3). - -check_no_multipart_uploads(Bucket, RiakPid) -> - HashBucket = stanchion_utils:to_bucket_name(objects, Bucket), - - {{ok, Keys}, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:list_keys(RiakPid, HashBucket)), - stanchion_stats:update([riakc, list_all_manifest_keys], TAT), - - %% check all up - lists:all(fun(Key) -> - GetResult = stanchion_utils:get_manifests_raw(RiakPid, Bucket, Key), - has_no_upload(GetResult) - end, Keys). - -has_no_upload({ok, Obj}) -> - Manifests = manifests_from_riak_object(Obj), - lists:all(fun({_UUID,Manifest}) -> - case Manifest?MANIFEST.state of - writing -> - %% if this is mp => false - not proplists:is_defined(multipart, Manifest?MANIFEST.props); - _ -> - true - end - end, Manifests); -has_no_upload({error, notfound}) -> true; -has_no_upload({error, _} = Error) -> - _ = lager:error("unexpected error: ~p", [Error]), - true. - --spec manifests_from_riak_object(riakc_obj:riakc_obj()) -> orddict:orddict(). -manifests_from_riak_object(RiakObject) -> - %% For example, riak_cs_manifest_fsm:get_and_update/4 may wish to - %% update the #riakc_obj without a roundtrip to Riak first. So we - %% need to see what the latest - Contents = try - %% get_update_value will return the updatevalue or - %% a single old original value. - [{riakc_obj:get_update_metadata(RiakObject), - riakc_obj:get_update_value(RiakObject)}] - catch throw:_ -> - %% Original value had many contents - riakc_obj:get_contents(RiakObject) - end, - DecodedSiblings = [binary_to_term(V) || - {_, V}=Content <- Contents, - not stanchion_utils:has_tombstone(Content)], - - %% Upgrade the manifests to be the latest erlang - %% record version - Upgraded = upgrade_wrapped_manifests(DecodedSiblings), - - %% resolve the siblings - stanchion_manifest_resolution:resolve(Upgraded). - --spec upgrade_wrapped_manifests([orddict:orddict()]) -> [orddict:orddict()]. -upgrade_wrapped_manifests(ListofOrdDicts) -> - DictMapFun = fun(_Key, Value) -> upgrade_manifest(Value) end, - MapFun = fun(Value) -> orddict:map(DictMapFun, Value) end, - lists:map(MapFun, ListofOrdDicts). - -%% @doc Upgrade the manifest to the most recent -%% version of the manifest record. This is so that -%% _most_ of the codebase only has to deal with -%% the most recent version of the record. --spec upgrade_manifest(lfs_manifest() | #lfs_manifest_v2{}) -> lfs_manifest(). -upgrade_manifest(#lfs_manifest_v2{block_size=BlockSize, - bkey=Bkey, - metadata=Metadata, - created=Created, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - state=State, - write_start_time=WriteStartTime, - last_block_written_time=LastBlockWrittenTime, - write_blocks_remaining=WriteBlocksRemaining, - delete_marked_time=DeleteMarkedTime, - last_block_deleted_time=LastBlockDeletedTime, - delete_blocks_remaining=DeleteBlocksRemaining, - acl=Acl, - props=Properties, - cluster_id=ClusterID}) -> - - upgrade_manifest(?MANIFEST{block_size=BlockSize, - bkey=Bkey, - metadata=Metadata, - created=Created, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - state=State, - write_start_time=WriteStartTime, - last_block_written_time=LastBlockWrittenTime, - write_blocks_remaining=WriteBlocksRemaining, - delete_marked_time=DeleteMarkedTime, - last_block_deleted_time=LastBlockDeletedTime, - delete_blocks_remaining=DeleteBlocksRemaining, - acl=Acl, - props=Properties, - cluster_id=ClusterID}); - -upgrade_manifest(?MANIFEST{props=Props}=M) -> - M?MANIFEST{props=fixup_props(Props)}. - --spec fixup_props(undefined | list()) -> list(). -fixup_props(undefined) -> - []; -fixup_props(Props) when is_list(Props) -> - Props. diff --git a/test/stanchion_config_test.erl b/test/stanchion_config_test.erl deleted file mode 100644 index 5a97ef1..0000000 --- a/test/stanchion_config_test.erl +++ /dev/null @@ -1,87 +0,0 @@ --module(stanchion_config_test). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). - -default_config_test() -> - Config = cuttlefish_unit:generate_templated_config(schema_files(), [], context()), - cuttlefish_unit:assert_config(Config, "stanchion.host", {"127.0.0.1", 8085}), - cuttlefish_unit:assert_config(Config, "stanchion.riak_host", {"127.0.0.1", 8087}), - cuttlefish_unit:assert_not_configured(Config, "stanchion.ssl"), - cuttlefish_unit:assert_config(Config, "stanchion.admin_key", "admin-key"), - cuttlefish_unit:assert_config(Config, "stanchion.admin_secret", "admin-secret"), - cuttlefish_unit:assert_config(Config, "stanchion.auth_bypass", false), - - {ok, [ConsoleLog, ErrorLog]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config), - cuttlefish_unit:assert_config([ConsoleLog], "lager_file_backend", [{file, "./log/console.log"}, - {level, info}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]), - cuttlefish_unit:assert_config([ErrorLog], "lager_file_backend", [{file, "./log/error.log"}, - {level, error}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]), - ok. - -ssl_config_test() -> - Conf = [{["ssl", "certfile"], "path/certfile"}, - {["ssl", "keyfile"], "path/keyfile"}], - Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), - cuttlefish_unit:assert_config(Config, "stanchion.ssl", [{keyfile, "path/keyfile"}, - {certfile, "path/certfile"}]), - ok. - -lager_syslog_test() -> - Conf = [{["log", "syslog"], on}, - {["log", "syslog", "ident"], "ident-test"}, - {["log", "syslog", "facility"], local7}, - {["log", "syslog", "level"], debug} - ], - Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), - cuttlefish_unit:assert_config(Config, "lager.handlers.lager_syslog_backend", ["ident-test", local7, debug]), - ok. - -lager_hander_test() -> - Conf = [ - {["log", "console", "file"], "./log/consolefile.log"}, - {["log", "console", "level"], "debug"}, - {["log", "console", "size"], "1MB"}, - {["log", "console", "rotation"], "$D5"}, - {["log", "console", "rotation", "keep"], "10"}, - {["log", "error", "file"], "./log/errorfile.log"}, - {["log", "error", "size"], "1KB"}, - {["log", "error", "rotation"], "$D10"}, - {["log", "error", "rotation", "keep"], "20"} - ], - Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), - {ok, [ConsoleLog, ErrorLog]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config), - cuttlefish_unit:assert_config([ConsoleLog], "lager_file_backend", [{file, "./log/consolefile.log"}, - {level, debug}, - {size, 1048576}, - {date, "$D5"}, - {count, 10}]), - cuttlefish_unit:assert_config([ErrorLog], "lager_file_backend", [{file, "./log/errorfile.log"}, - {level, error}, - {size, 1024}, - {date, "$D10"}, - {count, 20}]), - - CurrentConf1 = [{["log", "console", "rotation", "keep"], "current"}], - Config1 = cuttlefish_unit:generate_templated_config(schema_files(), CurrentConf1, context()), - {ok, [ConsoleLog1, _ErrorLog1]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config1), - cuttlefish_unit:assert_config([ConsoleLog1], "lager_file_backend.count", 0), - - CurrentConf2 = [{["log", "error", "rotation", "keep"], "current"}], - Config2 = cuttlefish_unit:generate_templated_config(schema_files(), CurrentConf2, context()), - {ok, [_ConsoleLog2, ErrorLog2]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config2), - cuttlefish_unit:assert_config([ErrorLog2], "lager_file_backend.count", 0), - ok. - -schema_files() -> - ["../rel/files/stanchion.schema"]. - -context() -> - {ok, Context} = file:consult("../rel/vars.config"), - Context. diff --git a/tools.mk b/tools.mk deleted file mode 100644 index 07dfdd7..0000000 --- a/tools.mk +++ /dev/null @@ -1,69 +0,0 @@ -REBAR ?= ./rebar - -compile-no-deps: - ${REBAR} compile skip_deps=true - -test: compile - ${REBAR} eunit skip_deps=true - -docs: - ${REBAR} doc skip_deps=true - -xref: compile - ${REBAR} xref skip_deps=true - -PLT ?= $(HOME)/.combo_dialyzer_plt -LOCAL_PLT = .local_dialyzer_plt -DIALYZER_FLAGS ?= -Wunmatched_returns - -${PLT}: compile - @if [ -f $(PLT) ]; then \ - dialyzer --check_plt --plt $(PLT) --apps $(DIALYZER_APPS) && \ - dialyzer --add_to_plt --plt $(PLT) --output_plt $(PLT) --apps $(DIALYZER_APPS) ; test $$? -ne 1; \ - else \ - dialyzer --build_plt --output_plt $(PLT) --apps $(DIALYZER_APPS); test $$? -ne 1; \ - fi - -${LOCAL_PLT}: compile - @if [ -d deps ]; then \ - if [ -f $(LOCAL_PLT) ]; then \ - dialyzer --check_plt --plt $(LOCAL_PLT) deps/*/ebin && \ - dialyzer --add_to_plt --plt $(LOCAL_PLT) --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ - else \ - dialyzer --build_plt --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ - fi \ - fi - -dialyzer-run: - @echo "==> $(shell basename $(shell pwd)) (dialyzer)" - @if [ -f $(LOCAL_PLT) ]; then \ - PLTS="$(PLT) $(LOCAL_PLT)"; \ - else \ - PLTS=$(PLT); \ - fi; \ - if [ -f dialyzer.ignore-warnings ]; then \ - if [ $$(grep -cvE '[^[:space:]]' dialyzer.ignore-warnings) -ne 0 ]; then \ - echo "ERROR: dialyzer.ignore-warnings contains a blank/empty line, this will match all messages!"; \ - exit 1; \ - fi; \ - dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin > dialyzer_warnings ; \ - egrep -v "^\s*(done|Checking|Proceeding|Compiling)" dialyzer_warnings | grep -F -f dialyzer.ignore-warnings -v > dialyzer_unhandled_warnings ; \ - cat dialyzer_unhandled_warnings ; \ - [ $$(cat dialyzer_unhandled_warnings | wc -l) -eq 0 ] ; \ - else \ - dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin; \ - fi - -dialyzer-quick: compile-no-deps dialyzer-run - -dialyzer: ${PLT} ${LOCAL_PLT} dialyzer-run - -cleanplt: - @echo - @echo "Are you sure? It takes several minutes to re-build." - @echo Deleting $(PLT) and $(LOCAL_PLT) in 5 seconds. - @echo - sleep 5 - rm $(PLT) - rm $(LOCAL_PLT) -