diff --git a/contrib/kyua/cli/cmd_test.cpp b/contrib/kyua/cli/cmd_test.cpp --- a/contrib/kyua/cli/cmd_test.cpp +++ b/contrib/kyua/cli/cmd_test.cpp @@ -33,6 +33,7 @@ #include "cli/common.ipp" #include "drivers/run_tests.hpp" #include "model/test_program.hpp" +#include "model/test_case.hpp" #include "model/test_result.hpp" #include "store/layout.hpp" #include "utils/cmdline/options.hpp" @@ -114,7 +115,11 @@ _ui->out(F("%s [%s]") % cli::format_result(result) % cli::format_delta(duration)); - type_count[result.type()]++; + const model::test_case& test_case = test_program.find(test_case_name); + auto ft = test_case.get_flaky_tracker(); + + if (!ft || ft->attempts_left() == 0) + type_count[result.type()]++; } }; diff --git a/contrib/kyua/drivers/run_tests.cpp b/contrib/kyua/drivers/run_tests.cpp --- a/contrib/kyua/drivers/run_tests.cpp +++ b/contrib/kyua/drivers/run_tests.cpp @@ -1,4 +1,4 @@ -// Copyright 2011 The Kyua Authors. +// Copyright 2025 The Kyua Authors. // All rights reserved. // // Redistribution and use in source and binary forms, with or without @@ -203,7 +203,16 @@ dynamic_cast< const scheduler::test_result_handle* >( result_handle.get()); - put_test_result(test_case_id, *test_result_handle, tx); + const model::test_program_ptr test_program = + test_result_handle->test_program(); + const model::test_case& test_case = test_program->find( + test_result_handle->test_case_name()); + auto ft = test_case.get_flaky_tracker(); + if (ft) + ft->attempt_taken(test_result_handle); + + if (!ft || ft->attempts_left() == 0) + put_test_result(test_case_id, *test_result_handle, tx); const model::test_result test_result = safe_cleanup(*test_result_handle); hooks.got_result( @@ -275,6 +284,7 @@ path_to_id_map ids_cache; pid_to_id_map in_flight; std::vector< engine::scan_result > exclusive_tests; + std::vector< engine::scan_result > flaky_tests; const std::size_t slots = user_config.lookup< config::positive_int_node >( "parallelism"); @@ -286,7 +296,13 @@ // first with the assumption that the spawning is faster than any single // job, so we want to keep as many jobs in the background as possible. while (in_flight.size() < slots) { - optional< engine::scan_result > match = scanner.yield(); + optional< engine::scan_result > match; + if (!flaky_tests.empty()) { + match = flaky_tests.back(); + flaky_tests.pop_back(); + } else { + match = scanner.yield(); + } if (!match) break; const model::test_program_ptr test_program = match.get().first; @@ -323,17 +339,40 @@ in_flight.erase(iter); finish_test(result_handle, test_case_id, tx, hooks); + + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + const model::test_program_ptr test_program = + test_result_handle->test_program(); + const model::test_case& test_case = test_program->find( + test_result_handle->test_case_name()); + auto ft = test_case.get_flaky_tracker(); + if (ft && ft->attempts_left() > 0) + flaky_tests.push_back(std::make_pair(test_program, test_case.name())); } - } while (!in_flight.empty() || !scanner.done()); + } while (!in_flight.empty() || !flaky_tests.empty() || !scanner.done()); // Run any exclusive tests that we spotted earlier sequentially. for (std::vector< engine::scan_result >::const_iterator iter = exclusive_tests.begin(); iter != exclusive_tests.end(); ++iter) { - const pid_and_id_pair data = start_test( - handle, *iter, tx, ids_cache, user_config, hooks); - scheduler::result_handle_ptr result_handle = handle.wait_any(); - finish_test(result_handle, data.second, tx, hooks); + engine::flaky::tracker_ptr ft = nullptr; + do { + const pid_and_id_pair data = start_test( + handle, *iter, tx, ids_cache, user_config, hooks); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + finish_test(result_handle, data.second, tx, hooks); + + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + const model::test_program_ptr test_program = + test_result_handle->test_program(); + const model::test_case& test_case = test_program->find( + test_result_handle->test_case_name()); + ft = test_case.get_flaky_tracker(); + } while (ft && ft->attempts_left() > 0); } tx.commit(); diff --git a/contrib/kyua/engine/atf_list.cpp b/contrib/kyua/engine/atf_list.cpp --- a/contrib/kyua/engine/atf_list.cpp +++ b/contrib/kyua/engine/atf_list.cpp @@ -125,6 +125,8 @@ mdbuilder.set_string("execenv", value); } else if (name == "execenv.jail.params") { mdbuilder.set_string("execenv_jail_params", value); + } else if (name == "flaky") { + mdbuilder.set_string("flaky", value); } else if (name == "is.exclusive") { mdbuilder.set_string("is_exclusive", value); } else if (name == "require.config") { diff --git a/contrib/kyua/engine/flaky/oneofn_tracker.hpp b/contrib/kyua/engine/flaky/oneofn_tracker.hpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky/oneofn_tracker.hpp @@ -0,0 +1,60 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/flaky/oneofn_tracker.hpp + +#if !defined(ENGINE_FLAKY_ONEOFN_TRACKER_HPP) +#define ENGINE_FLAKY_ONEOFN_TRACKER_HPP + +#include "engine/flaky/tracker.hpp" + + +namespace engine { + + +namespace flaky { + + +class oneofn_tracker : public tracker { + std::size_t _attempts_left = 0; +public: + oneofn_tracker() {} + + void init(const model::test_case&); + void attempt_taken(const scheduler::test_result_handle*); + std::size_t attempts_left() const; +}; + + +} // namespace flaky + + +} // namespace engine + + +#endif // !defined(ENGINE_FLAKY_ONEOFN_TRACKER_HPP) diff --git a/contrib/kyua/engine/flaky/oneofn_tracker.cpp b/contrib/kyua/engine/flaky/oneofn_tracker.cpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky/oneofn_tracker.cpp @@ -0,0 +1,79 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/flaky/oneofn_tracker.hpp" + +#include "engine/scheduler.hpp" +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_result.hpp" +#include "utils/format/macros.hpp" + + +void +engine::flaky::oneofn_tracker::init(const model::test_case& tc) +{ + auto flaky = tc.get_metadata().flaky(); + if (flaky.empty()) + throw engine::error("flaky metadata expected to be set"); + + try { + _attempts_left = std::stoull(flaky); + if (_attempts_left < 1) + throw std::exception{}; + } catch (const std::exception&) { + throw engine::error(F("Invalid flaky spec '%s'") % flaky); + } +} + + +void +engine::flaky::oneofn_tracker::attempt_taken( + const scheduler::test_result_handle* test_result_handle) +{ + const auto result = test_result_handle->test_result().type(); + if (result == model::test_result_passed || + result == model::test_result_expected_failure || + result == model::test_result_skipped) { + _attempts_left = 0; + return; + } + + if (_attempts_left < 1) + throw engine::error("_attempts_left expected to be > 0"); + + _attempts_left--; +} + + +std::size_t +engine::flaky::oneofn_tracker::attempts_left() const +{ + return _attempts_left; +} diff --git a/contrib/kyua/engine/flaky/tracker.hpp b/contrib/kyua/engine/flaky/tracker.hpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky/tracker.hpp @@ -0,0 +1,69 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/flaky/tracker.hpp + +#if !defined(ENGINE_FLAKY_TRACKER_HPP) +#define ENGINE_FLAKY_TRACKER_HPP + +#include +#include + +#include "engine/scheduler_fwd.hpp" +#include "model/test_case_fwd.hpp" + + +namespace engine { + + +namespace flaky { + + +class tracker { +public: + tracker() {} + virtual ~tracker() {} + + virtual void init(const model::test_case&) = 0; + virtual void attempt_taken(const scheduler::test_result_handle*) = 0; + virtual std::size_t attempts_left() const = 0; +}; + + +typedef std::shared_ptr< tracker > tracker_ptr; + +tracker_ptr make_tracker(const model::test_case&); + + +} // namespace flaky + + +} // namespace engine + + +#endif // !defined(ENGINE_FLAKY_TRACKER_HPP) diff --git a/contrib/kyua/engine/flaky/tracker.cpp b/contrib/kyua/engine/flaky/tracker.cpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky/tracker.cpp @@ -0,0 +1,59 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/flaky/tracker.hpp" + +#include "engine/flaky/oneofn_tracker.hpp" +#include "engine/flaky/xofy_tracker.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" + + +engine::flaky::tracker_ptr +engine::flaky::make_tracker(const model::test_case& tc) +{ + auto flaky = tc.get_metadata().flaky(); + if (flaky.empty()) + return nullptr; + + engine::flaky::tracker_ptr tracker; + + if (flaky.find('/') != std::string::npos) { + tracker = std::shared_ptr< engine::flaky::tracker >( + new engine::flaky::xofy_tracker() + ); + } else { + tracker = std::shared_ptr< engine::flaky::tracker >( + new engine::flaky::oneofn_tracker() + ); + } + + tracker->init(tc); + + return tracker; +} diff --git a/contrib/kyua/engine/flaky/xofy_tracker.hpp b/contrib/kyua/engine/flaky/xofy_tracker.hpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky/xofy_tracker.hpp @@ -0,0 +1,61 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/flaky/xofy_tracker.hpp + +#if !defined(ENGINE_FLAKY_XOFY_TRACKER_HPP) +#define ENGINE_FLAKY_XOFY_TRACKER_HPP + +#include "engine/flaky/tracker.hpp" + + +namespace engine { + + +namespace flaky { + + +class xofy_tracker : public tracker { + std::size_t _x; + std::size_t _y; +public: + xofy_tracker() {} + + void init(const model::test_case&); + void attempt_taken(const scheduler::test_result_handle*); + std::size_t attempts_left() const; +}; + + +} // namespace flaky + + +} // namespace engine + + +#endif // !defined(ENGINE_FLAKY_XOFY_TRACKER_HPP) diff --git a/contrib/kyua/engine/flaky/xofy_tracker.cpp b/contrib/kyua/engine/flaky/xofy_tracker.cpp new file mode 100644 --- /dev/null +++ b/contrib/kyua/engine/flaky/xofy_tracker.cpp @@ -0,0 +1,95 @@ +// Copyright 2025 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/flaky/xofy_tracker.hpp" + +#include "engine/scheduler.hpp" +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_result.hpp" +#include "utils/format/macros.hpp" + + +void +engine::flaky::xofy_tracker::init(const model::test_case& tc) +{ + auto flaky = tc.get_metadata().flaky(); + if (flaky.empty()) + throw engine::error("flaky metadata expected to be set"); + + try { + const std::size_t pos = flaky.find('/'); + if (pos == std::string::npos) + throw std::exception{}; + + const std::string left_str = flaky.substr(0, pos); + const std::string right_str = flaky.substr(pos + 1); + + if (left_str.empty() || right_str.empty()) + throw std::exception{}; + + _x = std::stoull(left_str); + _y = std::stoull(right_str); + + if (_x < 1 || _y < 1 || _x > _y) + throw std::exception{}; + } catch (const std::exception&) { + throw engine::error(F("Invalid flaky spec '%s'") % flaky); + } +} + + +void +engine::flaky::xofy_tracker::attempt_taken( + const scheduler::test_result_handle* test_result_handle) +{ + const auto result = test_result_handle->test_result().type(); + if (result == model::test_result_passed || + result == model::test_result_expected_failure || + result == model::test_result_skipped) { + _x--; + } + + if (_x == 0) { + _y = 0; + return; + } + + if (_y < 1) + throw engine::error("_y expected to be > 0"); + + _y--; +} + + +std::size_t +engine::flaky::xofy_tracker::attempts_left() const +{ + return _y; +} diff --git a/contrib/kyua/model/metadata.hpp b/contrib/kyua/model/metadata.hpp --- a/contrib/kyua/model/metadata.hpp +++ b/contrib/kyua/model/metadata.hpp @@ -69,6 +69,7 @@ const std::string& description(void) const; const std::string& execenv(void) const; const std::string& execenv_jail_params(void) const; + const std::string& flaky(void) const; bool has_cleanup(void) const; bool has_execenv(void) const; bool is_exclusive(void) const; @@ -116,6 +117,7 @@ metadata_builder& set_description(const std::string&); metadata_builder& set_execenv(const std::string&); metadata_builder& set_execenv_jail_params(const std::string&); + metadata_builder& set_flaky(const std::string&); metadata_builder& set_has_cleanup(const bool); metadata_builder& set_is_exclusive(const bool); metadata_builder& set_required_configs(const strings_set&); diff --git a/contrib/kyua/model/metadata.cpp b/contrib/kyua/model/metadata.cpp --- a/contrib/kyua/model/metadata.cpp +++ b/contrib/kyua/model/metadata.cpp @@ -250,6 +250,7 @@ tree.define< config::string_node >("description"); tree.define< config::string_node >("execenv"); tree.define< config::string_node >("execenv_jail_params"); + tree.define< config::string_node >("flaky"); tree.define< config::bool_node >("has_cleanup"); tree.define< config::bool_node >("is_exclusive"); tree.define< config::strings_set_node >("required_configs"); @@ -276,6 +277,7 @@ tree.set< config::string_node >("description", ""); tree.set< config::string_node >("execenv", ""); tree.set< config::string_node >("execenv_jail_params", ""); + tree.set< config::string_node >("flaky", ""); tree.set< config::bool_node >("has_cleanup", false); tree.set< config::bool_node >("is_exclusive", false); tree.set< config::strings_set_node >("required_configs", @@ -501,6 +503,20 @@ } +/// Returns flaky declaration string. +/// +/// \return The flaky declaration string. +const std::string& +model::metadata::flaky(void) const +{ + if (_pimpl->props.is_set("flaky")) { + return _pimpl->props.lookup< config::string_node >("flaky"); + } else { + return get_defaults().lookup< config::string_node >("flaky"); + } +} + + /// Returns whether the test has a cleanup part or not. /// /// \return True if there is a cleanup part; false otherwise. @@ -984,6 +1000,21 @@ } +/// Sets flaky declaration string. +/// +/// \param name The flaky declaration string. +/// +/// \return A reference to this builder. +/// +/// \throw model::error If the value is invalid. +model::metadata_builder& +model::metadata_builder::set_flaky(const std::string& name) +{ + set< config::string_node >(_pimpl->props, "flaky", name); + return *this; +} + + /// Sets whether the test has a cleanup part or not. /// /// \param cleanup True if the test has a cleanup part; false otherwise. diff --git a/contrib/kyua/model/test_case.hpp b/contrib/kyua/model/test_case.hpp --- a/contrib/kyua/model/test_case.hpp +++ b/contrib/kyua/model/test_case.hpp @@ -39,6 +39,7 @@ #include #include "engine/debugger.hpp" +#include "engine/flaky/tracker.hpp" #include "model/metadata_fwd.hpp" #include "model/test_result_fwd.hpp" #include "utils/noncopyable.hpp" @@ -75,6 +76,7 @@ void attach_debugger(engine::debugger_ptr) const; engine::debugger_ptr get_debugger() const; + engine::flaky::tracker_ptr get_flaky_tracker() const; bool operator==(const test_case&) const; bool operator!=(const test_case&) const; diff --git a/contrib/kyua/model/test_case.cpp b/contrib/kyua/model/test_case.cpp --- a/contrib/kyua/model/test_case.cpp +++ b/contrib/kyua/model/test_case.cpp @@ -63,6 +63,10 @@ /// Optional pointer to a debugger attached. engine::debugger_ptr debugger; + /// A flaky tracker attached. + engine::flaky::tracker_ptr flaky_tracker = nullptr; + bool flaky_tracker_initialized = false; + /// Constructor. /// /// \param name_ The name of the test case within the test program. @@ -254,6 +258,21 @@ } +/// Gets the optional flaky tracker. +/// +/// \return An optional pointer to a flaky tracker. +engine::flaky::tracker_ptr +model::test_case::get_flaky_tracker() const +{ + if (!_pimpl->flaky_tracker_initialized) { + _pimpl->flaky_tracker = engine::flaky::make_tracker(*this); + _pimpl->flaky_tracker_initialized = true; + } + + return _pimpl->flaky_tracker; +} + + /// Gets the fake result pre-stored for this test case. /// /// \return A fake result, or none if not defined. diff --git a/usr.bin/kyua/Makefile b/usr.bin/kyua/Makefile --- a/usr.bin/kyua/Makefile +++ b/usr.bin/kyua/Makefile @@ -118,6 +118,9 @@ engine/config.cpp \ engine/exceptions.cpp \ engine/filters.cpp \ + engine/flaky/tracker.cpp \ + engine/flaky/oneofn_tracker.cpp \ + engine/flaky/xofy_tracker.cpp \ engine/kyuafile.cpp \ engine/plain.cpp \ engine/requirements.cpp \