This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new e81df64ff Implement prop updates for shards
e81df64ff is described below
commit e81df64ffd2b68c62e2f3bdfc9086b552229ce13
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Sep 11 01:14:03 2025 -0400
Implement prop updates for shards
When we implemented partitioned dbs we added a generic `props` features to
the
db shards and the shard docs. The `props` is a generic prop (KV) list which
can
store database metadata properties. Currently it only stores the
`partitioned`
and the `hash` db properties.
Recently we discussed possibly storing a new TTL flag or moving some other
metadata bits like security or revs limits and such to props and we'd want
to
them both for the clustered and local shards (for local _dbs, _nodes etc).
In order to use props like that we'd want to allow dynamically updating
props
after the initial db creations so that's what this PR does.
We still want to ensure ``partitioned`` and hash ``properties`` are "static"
and we don't allow modifying them later so there an way in couch_db.erl to
flag
a set of properties as "static".
A part of the dynamic API to set props on shards was already implemented in
the
form of `couch_db_engine:set_props/2` so in the PR we just build the rest of
the bits in couch_db and fabric.
This is also a first part which update properties for shards files, we'll
follow up with another commit to allow update the props in the shard map
document as well.
---
src/couch/src/couch_db.erl | 40 ++++++++++++++++++++------
src/couch/src/couch_db_updater.erl | 20 +++++++++----
src/fabric/src/fabric.erl | 12 ++++++++
src/fabric/src/fabric_db_meta.erl | 25 +++++++++++++++-
src/fabric/src/fabric_rpc.erl | 4 +++
src/fabric/test/eunit/fabric_db_info_tests.erl | 38 +++++++++++++++++++++++-
6 files changed, 123 insertions(+), 16 deletions(-)
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 42d82204f..b8b8ee708 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -72,6 +72,9 @@
set_security/2,
set_user_ctx/2,
+ get_props/1,
+ update_props/3,
+
load_validation_funs/1,
reload_validation_funs/1,
@@ -157,6 +160,11 @@
% Purge client max lag window in seconds (defaulting to 24 hours)
-define(PURGE_LAG_SEC, 86400).
+% DB props which cannot be dynamically updated after db creation
+-define(PROP_PARTITIONED, partitioned).
+-define(PROP_HASH, hash).
+-define(STATIC_PROPS, [?PROP_PARTITIONED, ?PROP_HASH]).
+
start_link(Engine, DbName, Filepath, Options) ->
Arg = {Engine, DbName, Filepath, Options},
proc_lib:start_link(couch_db_updater, init, [Arg]).
@@ -232,9 +240,9 @@ is_clustered(#db{}) ->
is_clustered(?OLD_DB_REC = Db) ->
?OLD_DB_MAIN_PID(Db) == undefined.
-is_partitioned(#db{options = Options}) ->
- Props = couch_util:get_value(props, Options, []),
- couch_util:get_value(partitioned, Props, false).
+is_partitioned(#db{} = Db) ->
+ Props = get_props(Db),
+ couch_util:get_value(?PROP_PARTITIONED, Props, false).
close(#db{} = Db) ->
ok = couch_db_engine:decref(Db);
@@ -650,11 +658,7 @@ get_db_info(Db) ->
undefined -> null;
Else1 -> Else1
end,
- Props =
- case couch_db_engine:get_props(Db) of
- undefined -> null;
- Else2 -> {Else2}
- end,
+ Props = get_props(Db),
InfoList = [
{db_name, Name},
{engine, couch_db_engine:get_engine(Db)},
@@ -668,7 +672,7 @@ get_db_info(Db) ->
{disk_format_version, DiskVersion},
{committed_update_seq, CommittedUpdateSeq},
{compacted_seq, CompactedSeq},
- {props, Props},
+ {props, {Props}},
{uuid, Uuid}
],
{ok, InfoList}.
@@ -861,6 +865,24 @@ set_revs_limit(#db{main_pid = Pid} = Db, Limit) when Limit
> 0 ->
set_revs_limit(_Db, _Limit) ->
throw(invalid_revs_limit).
+get_props(#db{options = Options}) ->
+ couch_util:get_value(props, Options, []).
+
+update_props(#db{main_pid = Pid} = Db, K, V) ->
+ check_is_admin(Db),
+ case lists:member(K, ?STATIC_PROPS) of
+ true ->
+ throw({bad_request, <<"cannot update static property">>});
+ false ->
+ Props = get_props(Db),
+ Props1 =
+ case V of
+ undefined -> lists:keydelete(K, 1, Props);
+ _ -> lists:keystore(K, 1, Props, {K, V})
+ end,
+ gen_server:call(Pid, {set_props, Props1}, infinity)
+ end.
+
name(#db{name = Name}) ->
Name;
name(?OLD_DB_REC = Db) ->
diff --git a/src/couch/src/couch_db_updater.erl
b/src/couch/src/couch_db_updater.erl
index 909f1aeb5..1d0e177d5 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -97,6 +97,12 @@ handle_call({set_time_seq, TSeq}, _From, Db) ->
{ok, Db2} = couch_db_engine:commit_data(Db1#db{time_seq = TSeq}),
ok = couch_server:db_updated(Db2),
{reply, ok, Db2};
+handle_call({set_props, Props}, _From, Db) ->
+ {ok, Db1} = couch_db_engine:set_props(Db, Props),
+ Db2 = options_set_props(Db1, Props),
+ {ok, Db3} = couch_db_engine:commit_data(Db2),
+ ok = couch_server:db_updated(Db3),
+ {reply, ok, Db3};
handle_call({purge_docs, [], _}, _From, Db) ->
{reply, {ok, []}, Db};
handle_call({purge_docs, PurgeReqs0, Options}, _From, Db) ->
@@ -313,14 +319,18 @@ init_db(DbName, FilePath, EngineState, Options) ->
after_doc_read = ADR
},
- DbProps = couch_db_engine:get_props(InitDb),
-
- InitDb#db{
+ Db = InitDb#db{
committed_update_seq = couch_db_engine:get_update_seq(InitDb),
security = couch_db_engine:get_security(InitDb),
time_seq = couch_db_engine:get_time_seq(InitDb),
- options = lists:keystore(props, 1, NonCreateOpts, {props, DbProps})
- }.
+ options = NonCreateOpts
+ },
+ DbProps = couch_db_engine:get_props(Db),
+ options_set_props(Db, DbProps).
+
+options_set_props(#db{options = Options} = Db, Props) ->
+ Options1 = lists:keystore(props, 1, Options, {props, Props}),
+ Db#db{options = Options1}.
refresh_validate_doc_funs(#db{name = <<"shards/", _/binary>> = Name} = Db) ->
spawn(fabric, reset_validation_funs, [mem3:dbname(Name)]),
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 0a4b4de25..a2bf82482 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -25,6 +25,8 @@
get_db_info/1,
get_doc_count/1, get_doc_count/2,
set_revs_limit/3,
+ update_props/3,
+ update_props/4,
set_security/2, set_security/3,
get_revs_limit/1,
get_security/1, get_security/2,
@@ -186,6 +188,16 @@ get_revs_limit(DbName) ->
catch couch_db:close(Db)
end.
+%% @doc update shard property. Some properties like `partitioned` or `hash` are
+%% static and cannot be updated. They will return an error.
+-spec update_props(dbname(), atom() | binary(), any()) -> ok.
+update_props(DbName, K, V) ->
+ update_props(DbName, K, V, [?ADMIN_CTX]).
+
+-spec update_props(dbname(), atom() | binary(), any(), [option()]) -> ok.
+update_props(DbName, K, V, Options) when is_atom(K) orelse is_binary(K) ->
+ fabric_db_meta:update_props(dbname(DbName), K, V, opts(Options)).
+
%% @doc sets the readers/writers/admin permissions for a database
-spec set_security(dbname(), SecObj :: json_obj()) -> ok.
set_security(DbName, SecObj) ->
diff --git a/src/fabric/src/fabric_db_meta.erl
b/src/fabric/src/fabric_db_meta.erl
index 1013b958d..af4a069d4 100644
--- a/src/fabric/src/fabric_db_meta.erl
+++ b/src/fabric/src/fabric_db_meta.erl
@@ -16,7 +16,8 @@
set_revs_limit/3,
set_security/3,
get_all_security/2,
- set_purge_infos_limit/3
+ set_purge_infos_limit/3,
+ update_props/4
]).
-include_lib("fabric/include/fabric.hrl").
@@ -198,3 +199,25 @@ maybe_finish_get(#acc{workers = []} = Acc) ->
{stop, Acc};
maybe_finish_get(Acc) ->
{ok, Acc}.
+
+update_props(DbName, K, V, Options) ->
+ Shards = mem3:shards(DbName),
+ Workers = fabric_util:submit_jobs(Shards, update_props, [K, V, Options]),
+ Handler = fun handle_update_props_message/3,
+ Acc0 = {Workers, length(Workers) - 1},
+ case fabric_util:recv(Workers, #shard.ref, Handler, Acc0) of
+ {ok, ok} ->
+ ok;
+ {timeout, {DefunctWorkers, _}} ->
+ fabric_util:log_timeout(DefunctWorkers, "update_props"),
+ {error, timeout};
+ Error ->
+ Error
+ end.
+
+handle_update_props_message(ok, _, {_Workers, 0}) ->
+ {stop, ok};
+handle_update_props_message(ok, Worker, {Workers, Waiting}) ->
+ {ok, {lists:delete(Worker, Workers), Waiting - 1}};
+handle_update_props_message(Error, _, _Acc) ->
+ {error, Error}.
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index 18215ba34..492546b90 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -33,6 +33,7 @@
reset_validation_funs/1,
set_security/3,
set_revs_limit/3,
+ update_props/4,
create_shard_db_doc/2,
delete_shard_db_doc/2,
get_partition_info/2
@@ -269,6 +270,9 @@ set_revs_limit(DbName, Limit, Options) ->
set_purge_infos_limit(DbName, Limit, Options) ->
with_db(DbName, Options, {couch_db, set_purge_infos_limit, [Limit]}).
+update_props(DbName, K, V, Options) ->
+ with_db(DbName, Options, {couch_db, update_props, [K, V]}).
+
open_doc(DbName, DocId, Options) ->
with_db(DbName, Options, {couch_db, open_doc, [DocId, Options]}).
diff --git a/src/fabric/test/eunit/fabric_db_info_tests.erl
b/src/fabric/test/eunit/fabric_db_info_tests.erl
index e7df560a1..9a133ace5 100644
--- a/src/fabric/test/eunit/fabric_db_info_tests.erl
+++ b/src/fabric/test/eunit/fabric_db_info_tests.erl
@@ -20,7 +20,8 @@ main_test_() ->
fun setup/0,
fun teardown/1,
with([
- ?TDEF(t_update_seq_has_uuids)
+ ?TDEF(t_update_seq_has_uuids),
+ ?TDEF(t_update_and_get_props)
])
}.
@@ -55,3 +56,38 @@ t_update_seq_has_uuids(_) ->
?assertEqual(UuidFromShard, SeqUuid),
ok = fabric:delete_db(DbName, []).
+
+t_update_and_get_props(_) ->
+ DbName = ?tempdb(),
+ ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]),
+
+ {ok, Info} = fabric:get_db_info(DbName),
+ Props = couch_util:get_value(props, Info),
+ ?assertEqual({[]}, Props),
+
+ ?assertEqual(ok, fabric:update_props(DbName, <<"foo">>, 100)),
+ {ok, Info1} = fabric:get_db_info(DbName),
+ Props1 = couch_util:get_value(props, Info1),
+ ?assertEqual({[{<<"foo">>, 100}]}, Props1),
+
+ ?assertEqual(ok, fabric:update_props(DbName, bar, 101)),
+ {ok, Info2} = fabric:get_db_info(DbName),
+ Props2 = couch_util:get_value(props, Info2),
+ ?assertEqual(
+ {[
+ {<<"foo">>, 100},
+ {bar, 101}
+ ]},
+ Props2
+ ),
+
+ ?assertEqual(ok, fabric:update_props(DbName, <<"foo">>, undefined)),
+ {ok, Info3} = fabric:get_db_info(DbName),
+ ?assertEqual({[{bar, 101}]}, couch_util:get_value(props, Info3)),
+
+ Res = fabric:update_props(DbName, partitioned, true),
+ ?assertMatch({error, {bad_request, _}}, Res),
+ {ok, Info4} = fabric:get_db_info(DbName),
+ ?assertEqual({[{bar, 101}]}, couch_util:get_value(props, Info4)),
+
+ ok = fabric:delete_db(DbName, []).
