diff --git a/contrib/kyua/cli/cmd_test.cpp b/contrib/kyua/cli/cmd_test.cpp --- a/contrib/kyua/cli/cmd_test.cpp +++ b/contrib/kyua/cli/cmd_test.cpp @@ -32,7 +32,9 @@ #include "cli/common.ipp" #include "drivers/run_tests.hpp" +#include "engine/flaky_tracker.hpp" #include "model/test_program.hpp" +#include "model/test_case.hpp" #include "model/test_result.hpp" #include "store/layout.hpp" #include "utils/cmdline/options.hpp" @@ -114,7 +116,11 @@ _ui->out(F("%s [%s]") % cli::format_result(result) % cli::format_delta(duration)); - type_count[result.type()]++; + const model::test_case& test_case = test_program.find(test_case_name); + auto ft = test_case.get_flaky_tracker(); + + if (ft->attempts_left() == 0) + type_count[result.type()]++; } }; diff --git a/contrib/kyua/drivers/run_tests.cpp b/contrib/kyua/drivers/run_tests.cpp --- a/contrib/kyua/drivers/run_tests.cpp +++ b/contrib/kyua/drivers/run_tests.cpp @@ -1,4 +1,4 @@ -// Copyright 2011 The Kyua Authors. +// Copyright 2025 The Kyua Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without @@ -203,7 +203,15 @@ dynamic_cast< const scheduler::test_result_handle* >( result_handle.get()); - put_test_result(test_case_id, *test_result_handle, tx); + const model::test_program_ptr test_program = + test_result_handle->test_program(); + const model::test_case& test_case = test_program->find( + test_result_handle->test_case_name()); + auto ft = test_case.get_flaky_tracker(); + ft->attempt_taken(test_result_handle); + + if (ft->attempts_left() == 0) + put_test_result(test_case_id, *test_result_handle, tx); const model::test_result test_result = safe_cleanup(*test_result_handle); hooks.got_result( @@ -275,6 +283,7 @@ path_to_id_map ids_cache; pid_to_id_map in_flight; std::vector< engine::scan_result > exclusive_tests; + std::vector< engine::scan_result > flaky_tests; const std::size_t slots = user_config.lookup< config::positive_int_node >( "parallelism"); @@ -286,7 +295,13 @@ // first with the assumption that the spawning is faster than any single // job, so we want to keep as many jobs in the background as possible. while (in_flight.size() < slots) { - optional< engine::scan_result > match = scanner.yield(); + optional< engine::scan_result > match; + if (!flaky_tests.empty()) { + match = flaky_tests.back(); + flaky_tests.pop_back(); + } else { + match = scanner.yield(); + } if (!match) break; const model::test_program_ptr test_program = match.get().first; @@ -323,17 +338,40 @@ in_flight.erase(iter); finish_test(result_handle, test_case_id, tx, hooks); + + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + const model::test_program_ptr test_program = + test_result_handle->test_program(); + const model::test_case& test_case = test_program->find( + test_result_handle->test_case_name()); + auto ft = test_case.get_flaky_tracker(); + if (ft->attempts_left() > 0) + flaky_tests.push_back(std::make_pair(test_program, test_case.name())); } - } while (!in_flight.empty() || !scanner.done()); + } while (!in_flight.empty() || !flaky_tests.empty() || !scanner.done()); // Run any exclusive tests that we spotted earlier sequentially. for (std::vector< engine::scan_result >::const_iterator iter = exclusive_tests.begin(); iter != exclusive_tests.end(); ++iter) { - const pid_and_id_pair data = start_test( - handle, *iter, tx, ids_cache, user_config, hooks); - scheduler::result_handle_ptr result_handle = handle.wait_any(); - finish_test(result_handle, data.second, tx, hooks); + engine::flaky_tracker_ptr ft; + do { + const pid_and_id_pair data = start_test( + handle, *iter, tx, ids_cache, user_config, hooks); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + finish_test(result_handle, data.second, tx, hooks); + + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + const model::test_program_ptr test_program = + test_result_handle->test_program(); + const model::test_case& test_case = test_program->find( + test_result_handle->test_case_name()); + ft = test_case.get_flaky_tracker(); + } while (ft->attempts_left() > 0); } tx.commit(); diff --git a/contrib/kyua/engine/atf_list.cpp b/contrib/kyua/engine/atf_list.cpp --- a/contrib/kyua/engine/atf_list.cpp +++ b/contrib/kyua/engine/atf_list.cpp @@ -125,6 +125,8 @@ mdbuilder.set_string("execenv", value); } else if (name == "execenv.jail.params") { mdbuilder.set_string("execenv_jail_params", value); + } else if (name == "flaky") { + mdbuilder.set_string("flaky", value); } else if (name == "is.exclusive") { mdbuilder.set_string("is_exclusive", value); } else if (name == "require.config") { diff --git a/contrib/kyua/engine/flaky_tracker.hpp b/contrib/kyua/engine/flaky_tracker.hpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky_tracker.hpp @@ -0,0 +1,62 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/flaky_tracker.hpp + +#if !defined(FLAKY_TRACKER_HPP) +#define FLAKY_TRACKER_HPP + +#include +#include + +#include "engine/scheduler_fwd.hpp" +#include "model/test_case_fwd.hpp" + + +namespace engine { + + +class flaky_tracker { + std::size_t _attempts_left = 1; + +public: + flaky_tracker(const model::test_case&); + virtual ~flaky_tracker() {} + + void attempt_taken(const scheduler::test_result_handle*); + std::size_t attempts_left() const; +}; + + +typedef std::shared_ptr< flaky_tracker > flaky_tracker_ptr; + + +} // namespace engine + + +#endif // !defined(FLAKY_TRACKER_HPP) diff --git a/contrib/kyua/engine/flaky_tracker.cpp b/contrib/kyua/engine/flaky_tracker.cpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky_tracker.cpp @@ -0,0 +1,65 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/flaky_tracker.hpp" + +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_result.hpp" + + +engine::flaky_tracker::flaky_tracker(const model::test_case& tc) +{ + if (! tc.get_metadata().flaky().empty()) { + _attempts_left = std::stoul(tc.get_metadata().flaky()); + } +} + + +void +engine::flaky_tracker::attempt_taken( + const scheduler::test_result_handle* test_result_handle) +{ + const auto result = test_result_handle->test_result().type(); + if (result == model::test_result_passed || + result == model::test_result_expected_failure || + result == model::test_result_skipped) { + _attempts_left = 0; + return; + } + + _attempts_left--; +} + + +std::size_t +engine::flaky_tracker::attempts_left() const +{ + return _attempts_left; +} diff --git a/contrib/kyua/model/metadata.hpp b/contrib/kyua/model/metadata.hpp --- a/contrib/kyua/model/metadata.hpp +++ b/contrib/kyua/model/metadata.hpp @@ -69,6 +69,7 @@ const std::string& description(void) const; const std::string& execenv(void) const; const std::string& execenv_jail_params(void) const; + const std::string& flaky(void) const; bool has_cleanup(void) const; bool has_execenv(void) const; bool is_exclusive(void) const; @@ -116,6 +117,7 @@ metadata_builder& set_description(const std::string&); metadata_builder& set_execenv(const std::string&); metadata_builder& set_execenv_jail_params(const std::string&); + metadata_builder& set_flaky(const std::string&); metadata_builder& set_has_cleanup(const bool); metadata_builder& set_is_exclusive(const bool); metadata_builder& set_required_configs(const strings_set&); diff --git a/contrib/kyua/model/metadata.cpp b/contrib/kyua/model/metadata.cpp --- a/contrib/kyua/model/metadata.cpp +++ b/contrib/kyua/model/metadata.cpp @@ -250,6 +250,7 @@ tree.define< config::string_node >("description"); tree.define< config::string_node >("execenv"); tree.define< config::string_node >("execenv_jail_params"); + tree.define< config::string_node >("flaky"); tree.define< config::bool_node >("has_cleanup"); tree.define< config::bool_node >("is_exclusive"); tree.define< config::strings_set_node >("required_configs"); @@ -276,6 +277,7 @@ tree.set< config::string_node >("description", ""); tree.set< config::string_node >("execenv", ""); tree.set< config::string_node >("execenv_jail_params", ""); + tree.set< config::string_node >("flaky", ""); tree.set< config::bool_node >("has_cleanup", false); tree.set< config::bool_node >("is_exclusive", false); tree.set< config::strings_set_node >("required_configs", @@ -501,6 +503,20 @@ } +/// Returns flaky declaration string. +/// +/// \return The flaky declaration string. +const std::string& +model::metadata::flaky(void) const +{ + if (_pimpl->props.is_set("flaky")) { + return _pimpl->props.lookup< config::string_node >("flaky"); + } else { + return get_defaults().lookup< config::string_node >("flaky"); + } +} + + /// Returns whether the test has a cleanup part or not. /// /// \return True if there is a cleanup part; false otherwise. @@ -984,6 +1000,21 @@ } +/// Sets flaky declaration string. +/// +/// \param name The flaky declaration string. +/// +/// \return A reference to this builder. +/// +/// \throw model::error If the value is invalid. +model::metadata_builder& +model::metadata_builder::set_flaky(const std::string& name) +{ + set< config::string_node >(_pimpl->props, "flaky", name); + return *this; +} + + /// Sets whether the test has a cleanup part or not. /// /// \param cleanup True if the test has a cleanup part; false otherwise. diff --git a/contrib/kyua/model/test_case.hpp b/contrib/kyua/model/test_case.hpp --- a/contrib/kyua/model/test_case.hpp +++ b/contrib/kyua/model/test_case.hpp @@ -39,6 +39,7 @@ #include #include "engine/debugger.hpp" +#include "engine/flaky_tracker.hpp" #include "model/metadata_fwd.hpp" #include "model/test_result_fwd.hpp" #include "utils/noncopyable.hpp" @@ -75,6 +76,7 @@ void attach_debugger(engine::debugger_ptr) const; engine::debugger_ptr get_debugger() const; + engine::flaky_tracker_ptr get_flaky_tracker() const; bool operator==(const test_case&) const; bool operator!=(const test_case&) const; diff --git a/contrib/kyua/model/test_case.cpp b/contrib/kyua/model/test_case.cpp --- a/contrib/kyua/model/test_case.cpp +++ b/contrib/kyua/model/test_case.cpp @@ -63,6 +63,9 @@ /// Optional pointer to a debugger attached. engine::debugger_ptr debugger; + /// A flaky tracker attached. + engine::flaky_tracker_ptr flaky_tracker; + /// Constructor. /// /// \param name_ The name of the test case within the test program. @@ -254,6 +257,21 @@ } +/// Gets the optional flaky tracker. +/// +/// \return An optional pointer to a flaky tracker. +engine::flaky_tracker_ptr +model::test_case::get_flaky_tracker() const +{ + if (_pimpl->flaky_tracker) + return _pimpl->flaky_tracker; + + _pimpl->flaky_tracker = std::shared_ptr< engine::flaky_tracker >( + new engine::flaky_tracker(*this)); + return _pimpl->flaky_tracker; +} + + /// Gets the fake result pre-stored for this test case. /// /// \return A fake result, or none if not defined. diff --git a/usr.bin/kyua/Makefile b/usr.bin/kyua/Makefile --- a/usr.bin/kyua/Makefile +++ b/usr.bin/kyua/Makefile @@ -118,6 +118,7 @@ engine/config.cpp \ engine/exceptions.cpp \ engine/filters.cpp \ + engine/flaky_tracker.cpp \ engine/kyuafile.cpp \ engine/plain.cpp \ engine/requirements.cpp \