This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch improve-ci-runs in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0ce53c459b042cc4fe969107b4b07d04e36f930b Author: Nick Vatamaniuc <[email protected]> AuthorDate: Sun Dec 21 16:37:12 2025 -0500 Improve parallel test runs The main improvement is to separate the per-app makefile targets early, during templating phase, so they never share a common app structure during runtime. That lets us avoid reconfiguring the data directory at runtime, which effectively is an almost full restart internally since we're tearing down all the couch servers. So we're saving an extra app shutdown + restart cycle per app with the new scheme and improving the isolation. The new scheme uses the `tmp/$app` namespace for each app. After the tests run any unsucessful `tmp/$app` directories will be left behind so they can be inspected. This is especially useful for looking at the failed couch.log entries. There is a separate catlogs target to concatenate failed app couch.log files though maybe keeping them separately may be even easier to inspect. Since if app a failed, we'd just rather look at app a's logs and not app c's logs. Since setup_eunit rebar template now takes an app argument, opted to remove some older eunit targets keeping only the new ones supporting parallel execution. Otherwise, would have to add a special case statement in the plugin for no-app vs one app cases. Previously, `make eunit apps=a,b,c` did not run apps `a`, `b` and `c` as separate goals in parallel exection. They were passed in as the `a,b,c` goal. To fix that parse the apps into individual space separated goals, so they can be sped up by parallel execution as well. To keep the apps as isolated from each other as possible make build_eunit_config also create separate etc folders, use separate BUILDDIR macros and each apps only uses its own fixtures. Also use 0-valued ports the eunit template to avoid any chance of port collision and resulting flakiness. Rename SUBDIRS to EUNIT_SUBDIRS since they used for eunit tests currently only. Also, exclude docs and fauxton apps from the target list, since they are not proper erlang apps and don't have eunit tests. Newer Erlangs have `file:del_dir_r/1` so use that in cleanup_dirs to save a few lines of code. --- Makefile | 53 +++++-------------- rel/plugins/eunit_plugin.erl | 59 ++++++++++++++-------- setup_eunit.template | 27 +++++----- src/config/test/config_tests.erl | 2 +- src/couch/include/couch_eunit.hrl | 13 ++--- src/couch/src/test_util.erl | 51 +++---------------- src/couch_replicator/test/eunit/fixtures/logo.png | Bin 0 -> 3010 bytes 7 files changed, 81 insertions(+), 124 deletions(-) diff --git a/Makefile b/Makefile index 5c59f004c..4e9008b00 100644 --- a/Makefile +++ b/Makefile @@ -168,53 +168,26 @@ check: all @$(MAKE) nouveau-test ifdef apps -SUBDIRS = $(apps) +EUNIT_SUBDIRS = $(strip $(subst $(comma),$(space),$(apps))) else -SUBDIRS=$(shell ls src) +EUNIT_SUBDIRS = $(filter-out fauxton docs, $(shell ls src)) endif -# Used for comparing behaviour against he new `eunit` target, delete once we -# are happy with the new `eunit`. -.PHONY: old-eunit -old-eunit: export BUILDDIR = $(CURDIR) -old-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -old-eunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(CURDIR)/bin/couchjs $(CURDIR)/share/server/main.js -old-eunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 -old-eunit: - @COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) setup_eunit 2> /dev/null - @for dir in $(SUBDIRS); do \ - COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$$dir || exit 1; \ - done - # target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options -.PHONY: eunit $(SUBDIRS) -eunit: export BUILDDIR = $(CURDIR) +.PHONY: eunit $(EUNIT_SUBDIRS) eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config eunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(CURDIR)/bin/couchjs $(CURDIR)/share/server/main.js eunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 -eunit: ${SUBDIRS} - @cat tmp/couchdb-tests/*/log/couch.log > tmp/couch.log - @rm -rf tmp/couchdb-tests/* - -$(SUBDIRS): setup-eunit - @COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$@ #|| exit 1 - -setup-eunit: export BUILDDIR = $(CURDIR) -setup-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -setup-eunit: - @$(REBAR) setup_eunit 2> /dev/null - -just-eunit: export BUILDDIR = $(CURDIR) -just-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -just-eunit: - @$(REBAR) -r eunit $(EUNIT_OPTS) - -.PHONY: soak-eunit -soak-eunit: export BUILDDIR = $(CURDIR) -soak-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -soak-eunit: couch-core - @$(REBAR) setup_eunit 2> /dev/null - while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done +eunit: ${EUNIT_SUBDIRS} + +$(EUNIT_SUBDIRS): + @rm -rf tmp/$@ + @$(REBAR) setup_eunit app=$@ >/dev/null + @BUILDDIR=$(CURDIR)/tmp/$@ COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$@ && rm -rf tmp/$@ + +# cat together couch_log files after running eunit test +.PHONY: catlogs + @ls tmp/*/couch.log 2>/dev/null | xargs cat > tmp/couch.log || true # target: erlfmt-check - Check Erlang source code formatting erlfmt-check: diff --git a/rel/plugins/eunit_plugin.erl b/rel/plugins/eunit_plugin.erl index 356f4e0f7..eeb8ebe9e 100644 --- a/rel/plugins/eunit_plugin.erl +++ b/rel/plugins/eunit_plugin.erl @@ -25,37 +25,56 @@ is_base_dir(RebarConf) -> filename:absname(rebar_utils:get_cwd()) =:= rebar_config:get_xconf(RebarConf, base_dir, undefined). -build_eunit_config(Config0, AppFile) -> +build_eunit_config(Config, AppFile) -> Cwd = filename:absname(rebar_utils:get_cwd()), - DataDir = Cwd ++ "/tmp/data", - ViewIndexDir = Cwd ++ "/tmp/data", - StateDir = Cwd ++ "/tmp/data", - TmpDataDir = Cwd ++ "/tmp/tmp_data", - LogDir = Cwd ++ "/tmp", - cleanup_dirs([DataDir, TmpDataDir]), - Config1 = rebar_config:set_global(Config0, template, "setup_eunit"), - Config2 = rebar_config:set_global(Config1, prefix, Cwd), + App = rebar_config:get_global(Config, app, undefined), + case is_list(App) of + true -> ok; + false -> error(app_parameter_must_be_defined) + end, + case re:run(App, "^[_a-z0-9]+$") of + nomatch -> + error({app_parameter_must_be_just_one_app, App}); + {match, _} -> + Prefix = Cwd ++ "/tmp/" ++ App, + DataDir = Prefix ++ "/data", + ViewIndexDir = Prefix ++ "/data", + StateDir = Prefix ++ "/data", + TmpDataDir = Prefix ++ "/tmp_data", + EtcDir = Prefix ++ "/etc", + LogDir = Prefix, + build_config( + Config, + AppFile, + Prefix, + DataDir, + ViewIndexDir, + StateDir, + TmpDataDir, + EtcDir, + LogDir + ) + end. + +build_config(Config, AppFile, Prefix, DataDir, ViewIndexDir, StateDir, TmpDataDir, EtcDir, LogDir) -> + cleanup_dirs([DataDir, TmpDataDir, EtcDir]), + Config1 = rebar_config:set_global(Config, template, "setup_eunit"), + Config2 = rebar_config:set_global(Config1, prefix, Prefix), Config3 = rebar_config:set_global(Config2, data_dir, DataDir), Config4 = rebar_config:set_global(Config3, view_index_dir, ViewIndexDir), Config5 = rebar_config:set_global(Config4, log_dir, LogDir), - Config = rebar_config:set_global(Config5, state_dir, StateDir), - rebar_templater:create(Config, AppFile). + Config6 = rebar_config:set_global(Config5, etc_dir, EtcDir), + Config7 = rebar_config:set_global(Config6, tmp_data, TmpDataDir), + Config8 = rebar_config:set_global(Config7, state_dir, StateDir), + rebar_templater:create(Config8, AppFile). cleanup_dirs(Dirs) -> lists:foreach( fun(Dir) -> case filelib:is_dir(Dir) of - true -> del_dir(Dir); + true -> file:del_dir_r(Dir); false -> ok end end, Dirs ). - -del_dir(Dir) -> - All = filelib:wildcard(Dir ++ "/**"), - {Dirs, Files} = lists:partition(fun filelib:is_dir/1, All), - ok = lists:foreach(fun file:delete/1, Files), - SortedDirs = lists:sort(fun(A, B) -> length(A) > length(B) end, Dirs), - ok = lists:foreach(fun file:del_dir/1, SortedDirs), - ok = file:del_dir(Dir). diff --git a/setup_eunit.template b/setup_eunit.template index 11eee4458..08705b3bf 100644 --- a/setup_eunit.template +++ b/setup_eunit.template @@ -1,21 +1,24 @@ {variables, [ {package_author_name, "The Apache Software Foundation"}, - {cluster_port, 5984}, - {backend_port, 5986}, - {prometheus_port, 17986}, + {cluster_port, 0}, + {backend_port, 0}, + {prometheus_port, 0}, {node_name, "-name [email protected]"}, {data_dir, "/tmp"}, {prefix, "/tmp"}, {view_index_dir, "/tmp"}, {state_dir, "/tmp"}, - {log_dir, "/tmp"} + {log_dir, "/tmp"}, + {etc_dir, "/tmp"}, + {tmp_data, "/tmp"} ]}. -{dir, "tmp"}. -{dir, "tmp/etc"}. -{dir, "tmp/data"}. -{dir, "tmp/tmp_data"}. -{template, "rel/overlay/etc/default.ini", "tmp/etc/default_eunit.ini"}. -{template, "rel/overlay/etc/local.ini", "tmp/etc/local_eunit.ini"}. -{template, "rel/files/eunit.ini", "tmp/etc/eunit.ini"}. -{template, "rel/overlay/etc/vm.args", "tmp/etc/vm.args"}. +{dir, "{{prefix}}"}. +{dir, "{{etc_dir}}"}. +{dir, "{{data_dir}}"}. +{dir, "{{tmp_data}}"}. + +{template, "rel/overlay/etc/default.ini", "{{prefix}}/etc/default_eunit.ini"}. +{template, "rel/overlay/etc/local.ini", "{{prefix}}/etc/local_eunit.ini"}. +{template, "rel/files/eunit.ini", "{{prefix}}/etc/eunit.ini"}. +{template, "rel/overlay/etc/vm.args", "{{prefix}}/etc/vm.args"}. diff --git a/src/config/test/config_tests.erl b/src/config/test/config_tests.erl index 4e3bf60f7..301fbf7a6 100644 --- a/src/config/test/config_tests.erl +++ b/src/config/test/config_tests.erl @@ -24,7 +24,7 @@ -define(RESTART_TIMEOUT_IN_MILLISEC, 3000). -define(CONFIG_FIXTURESDIR, - filename:join([?BUILDDIR(), "src", "config", "test", "fixtures"]) + ?ABS_PATH("test/fixtures") ). -define(CONFIG_DEFAULT_TEST, diff --git a/src/couch/include/couch_eunit.hrl b/src/couch/include/couch_eunit.hrl index 2c10e257d..7d8686000 100644 --- a/src/couch/include/couch_eunit.hrl +++ b/src/couch/include/couch_eunit.hrl @@ -25,15 +25,13 @@ end end). -define(CONFIG_DEFAULT, - filename:join([?BUILDDIR(), "tmp", "etc", "default_eunit.ini"])). + filename:join([?BUILDDIR(), "etc", "default_eunit.ini"])). -define(CONFIG_CHAIN, [ ?CONFIG_DEFAULT, - filename:join([?BUILDDIR(), "tmp", "etc", "local_eunit.ini"]), - filename:join([?BUILDDIR(), "tmp", "etc", "eunit.ini"])]). --define(FIXTURESDIR, - filename:join([?BUILDDIR(), "src", "couch", "test", "eunit", "fixtures"])). + filename:join([?BUILDDIR(), "etc", "local_eunit.ini"]), + filename:join([?BUILDDIR(), "etc", "eunit.ini"])]). -define(TEMPDIR, - filename:join([?BUILDDIR(), "tmp", "tmp_data"])). + filename:join([?BUILDDIR(), "tmp_data"])). -define(APPDIR, filename:dirname(element(2, file:get_cwd()))). %% Account for the fact that source files are in src/<app>/.eunit/<module>.erl @@ -41,6 +39,9 @@ -define(ABS_PATH(File), %% src/<app>/.eunit/<module>.erl filename:join([?APPDIR, File])). +-define(FIXTURESDIR, + ?ABS_PATH("test/eunit/fixtures")). + -define(tempfile, fun() -> Suffix = couch_uuids:random(), diff --git a/src/couch/src/test_util.erl b/src/couch/src/test_util.erl index 96d1b280a..2b4583197 100644 --- a/src/couch/src/test_util.erl +++ b/src/couch/src/test_util.erl @@ -44,7 +44,7 @@ -include("couch_db_int.hrl"). -include("couch_bt_engine.hrl"). --record(test_context, {mocked = [], started = [], module, dir}). +-record(test_context, {mocked = [], started = [], module}). -define(DEFAULT_APPS, [inets, ibrowse, ssl, config, couch_epi, couch_event, couch]). @@ -81,61 +81,22 @@ start_couch() -> start_couch(ExtraApps) -> start_couch(?CONFIG_CHAIN, ExtraApps). -% This function starts CouchDB with optional extra apps in a dedicated -% directory under tmp/couchdb-tests/<uuid>/ — That is, all instances -% run isolated so that if more than one runs at a time, it does not -% interfere with any of the others. -% The sub-directories here are etc/ log/ and data/ for configuration -% logs and databases and view indexes respectively. -% The function copies the initial bespoke test-ini files to the new -% random directory and configures CouchDB to use those. -% The two pieces of state created here are the random dir in the file system and -% the process env var {config, ini_files} that is read by couch_config on -% startup. These need to be reset at the right points in time. -% the random dir is deleted and the env var reset when CouchDB is stopped. -% Note: there is currently a case where stop_couch/0 could be called and -% the file system cleanup can’t be run because we don’t get passed the test -% context with the random directory value. At the moment stop_couch/0 is not -% used anywhere, tho. Maybe we should remove it? start_couch(IniFiles, ExtraApps) -> - RandomDir = filename:join([builddir(), "tmp", "couchdb-tests", couch_uuids:random()]), - RandomEtcDir = filename:join([RandomDir, "etc"]), - RandomDataDir = ?b2l(filename:join([RandomDir, "data"])), - RandomLogFile = ?b2l(filename:join([RandomDir, "log", "couch.log"])), - - ok = filelib:ensure_path(RandomDir), - ok = filelib:ensure_path(RandomEtcDir), - ok = filelib:ensure_path(RandomDataDir), - ok = filelib:ensure_dir(RandomLogFile), - - RandomIniFiles = lists:map( - fun(SourceFile) -> - TargetFileName = lists:last(filename:split(SourceFile)), - TargetFile = filename:join([RandomEtcDir, TargetFileName]), - {ok, _} = file:copy(SourceFile, TargetFile), - ?b2l(TargetFile) - end, - IniFiles - ), load_applications_with_stats(), - ok = application:set_env(config, ini_files, RandomIniFiles), + ok = application:set_env(config, ini_files, IniFiles), Apps = start_applications(?DEFAULT_APPS ++ ExtraApps), - - ok = config:set("couchdb", "database_dir", RandomDataDir, false), - ok = config:set("couchdb", "view_index_dir", RandomDataDir, false), - ok = config:set("log", "file", RandomLogFile, false), ok = config:delete("compactions", "_default", false), - #test_context{started = Apps, dir = RandomDir}. + #test_context{started = Apps}. stop_couch() -> ok = stop_applications(?DEFAULT_APPS). -stop_couch(#test_context{started = Apps, dir = RandomDir}) -> - % deletion now happens in Makefile +stop_couch(#test_context{started = Apps}) -> % file:del_dir_r(RandomDir), stop_applications(Apps); stop_couch(_) -> stop_couch(). + with_couch_server_restart(Fun) -> Servers = couch_server:names(), test_util:with_processes_restart(Servers, Fun). @@ -415,7 +376,7 @@ mock(couch_stats) -> ok. load_applications_with_stats() -> - Wildcard = filename:join([?BUILDDIR(), "src/*/priv/stats_descriptions.cfg"]), + Wildcard = ?ABS_PATH("../*/priv/stats_descriptions.cfg"), [application:load(stats_file_to_app(File)) || File <- filelib:wildcard(Wildcard)], ok. diff --git a/src/couch_replicator/test/eunit/fixtures/logo.png b/src/couch_replicator/test/eunit/fixtures/logo.png new file mode 100644 index 000000000..d21ac025b Binary files /dev/null and b/src/couch_replicator/test/eunit/fixtures/logo.png differ
