diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7f46932..42b5ae1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,5 @@ enable_testing() add_subdirectory(core) -add_subdirectory(orm) \ No newline at end of file +add_subdirectory(orm) +add_subdirectory(benchmark) diff --git a/test/benchmark/CMakeLists.txt b/test/benchmark/CMakeLists.txt new file mode 100644 index 0000000..eb3514c --- /dev/null +++ b/test/benchmark/CMakeLists.txt @@ -0,0 +1,35 @@ +CPMAddPackage("gh:catchorg/Catch2@3.14.0") + +list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras) + +add_executable(Benchmarks + sql/StatementCacheBenchmark.cpp + ../orm/backend/test_backend_service.cpp + ../orm/backend/test_backend_service.hpp + ../orm/backend/test_connection.cpp + ../orm/backend/test_connection.hpp + ../orm/backend/test_parameter_binder.cpp + ../orm/backend/test_parameter_binder.hpp + ../orm/backend/test_result_reader.cpp + ../orm/backend/test_result_reader.hpp + ../orm/backend/test_statement.cpp + ../orm/backend/test_statement.hpp + ../utils/MetricsObserver.hpp + ../utils/MetricsObserver.cpp + ../utils/RecordingObserver.hpp + ../utils/RecordingObserver.cpp +) + +target_link_libraries(Benchmarks matador-orm matador-core Catch2::Catch2WithMain) + +target_include_directories(Benchmarks + PUBLIC + PRIVATE $/test +) + +#if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") +# target_compile_options(CoreTests PRIVATE -coverage) +# target_link_options(CoreTests PRIVATE -coverage) +#endif () + +add_test(NAME Benchmarks COMMAND Benchmarks) diff --git a/test/benchmark/sql/StatementCacheBenchmark.cpp b/test/benchmark/sql/StatementCacheBenchmark.cpp new file mode 100644 index 0000000..2033935 --- /dev/null +++ b/test/benchmark/sql/StatementCacheBenchmark.cpp @@ -0,0 +1,84 @@ +#include + +#include "matador/sql/connection_pool.hpp" +#include "matador/sql/interface/connection_impl.hpp" + +#include "../test/orm/backend/test_backend_service.hpp" + +#include "utils/MetricsObserver.hpp" +#include "utils/RecordingObserver.hpp" + +#include + +using namespace matador::test; +using namespace matador::sql; +using namespace matador::utils; + +TEST_CASE("Multithreaded stress test", "[statement][cache][stress]") { + backend_provider::instance().register_backend("noop", std::make_unique()); + + constexpr int thread_count = 16; + constexpr int iterations = 1000; + constexpr int sql_pool_size = 10; + + std::vector contexts; + for (int i = 0; i < sql_pool_size; ++i) { + const auto sql = "SELECT " + std::to_string(i); + contexts.push_back({sql, std::hash{}(sql)}); + } + + connection_pool pool("noop://noop.db", 4); + message_bus bus; + statement_cache cache(bus, pool, 5); + RecordingObserver observer(bus); + MetricsObserver metrics(bus); + + auto start_time = std::chrono::steady_clock::now(); + + std::atomic_int lock_failed_count{0}; + std::atomic_int exec_failed_count{0}; + + auto worker = [&](const int tid) { + std::mt19937 rng(tid); + std::uniform_int_distribution dist(0, sql_pool_size - 1); + + for (int i = 0; i < iterations; ++i) { + const auto& ctx = contexts[dist(rng)]; + if (const auto result = cache.acquire(ctx); !result) { + FAIL("Failed to acquire statement"); + } else { + if (const auto exec_result = result->execute(); !exec_result) { + if (exec_result.err().ec() == error_code::StatementLocked) { + ++lock_failed_count; + } else { + ++exec_failed_count; + } + } + } + } + }; + + std::vector threads; + for (int i = 0; i < thread_count; ++i) { + threads.emplace_back(worker, i); + } + for (auto& t : threads) { + t.join(); + } + const auto end_time = std::chrono::steady_clock::now(); + const auto duration = std::chrono::duration_cast(end_time - start_time); + + std::cout << "Executed " << (thread_count * iterations) << " statements in " << duration.count() << " ms (lock failed: " << lock_failed_count << ", execute failed: " << exec_failed_count << ")\n"; + std::cout << "Average lock wait time: " << metrics.get_average_lock_wait_time().count() << "ms\n"; + std::cout << "Total lock wait time: " << metrics.get_total_lock_wait_time().count() << "ms\n"; + std::cout << "Average execution time: " << metrics.get_average_execution_time().count() << "ms\n"; + std::cout << "Total execution time: " << metrics.get_total_execution_time().count() << "ms\n"; + std::cout << "Number of lock failures: " << metrics.get_lock_failure_count() << "\n"; + + // Some events should be generated + int accessed = 0; + while (auto e = observer.poll()) { + if (e->is()) accessed++; + } + REQUIRE(accessed > 0); +} diff --git a/test/orm/CMakeLists.txt b/test/orm/CMakeLists.txt index c663e3f..53f22eb 100644 --- a/test/orm/CMakeLists.txt +++ b/test/orm/CMakeLists.txt @@ -32,10 +32,17 @@ add_executable(OrmTests sql/StatementCacheTest.cpp utils/auto_reset_event.cpp utils/auto_reset_event.hpp + ../utils/RecordingObserver.hpp + ../utils/RecordingObserver.cpp ) target_link_libraries(OrmTests matador-orm matador-core Catch2::Catch2WithMain) +target_include_directories(OrmTests + PRIVATE $/test + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} +) + #target_compile_options(OrmTests PRIVATE -coverage) #target_link_options(OrmTests PRIVATE -coverage) diff --git a/test/orm/sql/StatementCacheTest.cpp b/test/orm/sql/StatementCacheTest.cpp index f7775e2..ca2683f 100644 --- a/test/orm/sql/StatementCacheTest.cpp +++ b/test/orm/sql/StatementCacheTest.cpp @@ -1,116 +1,17 @@ -#include #include #include "matador/sql/connection_pool.hpp" #include "matador/sql/error_code.hpp" -#include "matador/sql/statement_cache.hpp" - -#include "matador/utils/message_bus.hpp" +#include "matador/sql/interface/connection_impl.hpp" #include "../backend/test_backend_service.hpp" -#include "ConnectionPoolFixture.hpp" - -#include -#include -#include +#include "utils/RecordingObserver.hpp" using namespace matador::test; using namespace matador::sql; using namespace matador::utils; -class MetricsObserver { -public: - explicit MetricsObserver(message_bus &bus) { - subscriptions.push_back(bus.subscribe([this](const statement_lock_failed_event &ev) { - std::lock_guard lock(mutex_); - lock_failure_count_++; - total_lock_wait_time_ += ev.duration; - })); - subscriptions.push_back(bus.subscribe([this](const statement_execution_event &ev) { - std::lock_guard lock(mutex_); - execution_count_++; - total_execution_time_ += ev.duration; - })); - - } - - std::chrono::milliseconds get_average_lock_wait_time() const { - std::lock_guard lock(mutex_); - if (lock_failure_count_ == 0) { - return std::chrono::milliseconds{0}; - } - const auto millis = std::chrono::duration_cast(total_lock_wait_time_); - return std::chrono::milliseconds(millis.count() / lock_failure_count_); - } - - std::chrono::milliseconds get_average_execution_time() const { - std::lock_guard lock(mutex_); - if (execution_count_ == 0) { - return std::chrono::milliseconds{0}; - } - - const auto millis = std::chrono::duration_cast(total_execution_time_); - return std::chrono::milliseconds(millis.count() / execution_count_); - } - - std::chrono::milliseconds get_total_lock_wait_time() const { - std::lock_guard lock(mutex_); - return std::chrono::duration_cast(total_lock_wait_time_); - } - - std::chrono::milliseconds get_total_execution_time() const { - std::lock_guard lock(mutex_); - return std::chrono::duration_cast(total_execution_time_); - } - - size_t get_lock_failure_count() const { - std::lock_guard lock(mutex_); - return lock_failure_count_; - } - -private: - std::vector subscriptions; - mutable std::mutex mutex_; - size_t lock_failure_count_{0}; - size_t execution_count_{0}; - std::chrono::nanoseconds total_lock_wait_time_{0}; - std::chrono::nanoseconds total_execution_time_{0}; -}; - -class RecordingObserver final { -public: - explicit RecordingObserver(message_bus &bus) { - subscriptions.push_back(bus.subscribe([this](const statement_accessed_event &ev) { - std::lock_guard lock(mutex); - events.push(message::from_ref(ev)); - })); - subscriptions.push_back(bus.subscribe([this](const statement_added_event &ev) { - std::lock_guard lock(mutex); - events.push(message::from_ref(ev)); - })); - subscriptions.push_back(bus.subscribe([this](const statement_evicted_event &ev) { - std::lock_guard lock(mutex); - events.push(message::from_ref(ev)); - })); - } - - std::optional poll() { - std::lock_guard lock(mutex); - if (events.empty()) { - return std::nullopt; - } - auto evt = events.front(); - events.pop(); - return evt; - } - -private: - std::vector subscriptions; - std::mutex mutex; - std::queue events; -}; - TEST_CASE("Test statement cache", "[statement][cache]") { backend_provider::instance().register_backend("noop", std::make_unique()); @@ -222,77 +123,6 @@ TEST_CASE("Test statement reuse avoids reprepare", "[statement][cache][prepare]" auto stmt2 = result.value(); } -// TEST_CASE("Multithreaded stress test", "[statement][cache][stress]") { -// backend_provider::instance().register_backend("noop", std::make_unique()); -// -// constexpr int thread_count = 16; -// constexpr int iterations = 1000; -// constexpr int sql_pool_size = 10; -// -// std::vector sqls; -// for (int i = 0; i < sql_pool_size; ++i) { -// sqls.push_back("SELECT " + std::to_string(i)); -// } -// -// connection_pool pool("noop://noop.db", 4); -// message_bus bus; -// statement_cache cache(bus, pool, 5); -// RecordingObserver observer(bus); -// MetricsObserver metrics(bus); -// -// auto start_time = std::chrono::steady_clock::now(); -// -// std::atomic_int lock_failed_count{0}; -// std::atomic_int exec_failed_count{0}; -// -// auto worker = [&](const int tid) { -// std::mt19937 rng(tid); -// std::uniform_int_distribution dist(0, sql_pool_size - 1); -// -// for (int i = 0; i < iterations; ++i) { -// const auto& sql = sqls[dist(rng)]; -// if (const auto result = cache.acquire({sql}); !result) { -// FAIL("Failed to acquire statement"); -// } else { -// if (const auto exec_result = result->execute(); !exec_result) { -// if (exec_result.err().ec() == error_code::STATEMENT_LOCKED) { -// ++lock_failed_count; -// } else { -// ++exec_failed_count; -// } -// } -// } -// } -// }; -// -// std::vector threads; -// for (int i = 0; i < thread_count; ++i) { -// threads.emplace_back(worker, i); -// } -// -// for (auto& t : threads) { -// t.join(); -// } -// -// auto end_time = std::chrono::steady_clock::now(); -// auto duration = std::chrono::duration_cast(end_time - start_time); -// -// std::cout << "[Performance] Executed " << (thread_count * iterations) << " statements in " << duration.count() << " ms (lock failed: " << lock_failed_count << ", execute failed: " << exec_failed_count << ")\n"; -// -// std::cout << "Average lock wait time: " << metrics.get_average_lock_wait_time().count() << "ms\n"; -// std::cout << "Total lock wait time: " << metrics.get_total_lock_wait_time().count() << "ms\n"; -// std::cout << "Average execution time: " << metrics.get_average_execution_time().count() << "ms\n"; -// std::cout << "Total execution time: " << metrics.get_total_execution_time().count() << "ms\n"; -// std::cout << "Number of lock failures: " << metrics.get_lock_failure_count() << "\n"; -// -// // Some events should be generated -// int accessed = 0; -// while (auto e = observer.poll()) { -// if (e->is()) accessed++; -// } -// REQUIRE(accessed > 0); -// } - TEST_CASE("Race condition simulation with mixed access", "[statement_cache][race]") { backend_provider::instance().register_backend("noop", std::make_unique()); diff --git a/test/utils/MetricsObserver.cpp b/test/utils/MetricsObserver.cpp new file mode 100644 index 0000000..87d25f6 --- /dev/null +++ b/test/utils/MetricsObserver.cpp @@ -0,0 +1,51 @@ +#include "MetricsObserver.hpp" + +namespace matador::test { +MetricsObserver::MetricsObserver(utils::message_bus& bus) { + subscriptions.push_back(bus.subscribe([this](const sql::statement_lock_failed_event &ev) { + std::lock_guard lock(mutex_); + lock_failure_count_++; + total_lock_wait_time_ += ev.duration; + })); + subscriptions.push_back(bus.subscribe([this](const sql::statement_execution_event &ev) { + std::lock_guard lock(mutex_); + execution_count_++; + total_execution_time_ += ev.duration; + })); + +} + +std::chrono::milliseconds MetricsObserver::get_average_lock_wait_time() const { + std::lock_guard lock(mutex_); + if (lock_failure_count_ == 0) { + return std::chrono::milliseconds{0}; + } + const auto millis = std::chrono::duration_cast(total_lock_wait_time_); + return std::chrono::milliseconds(millis.count() / lock_failure_count_); +} + +std::chrono::milliseconds MetricsObserver::get_average_execution_time() const { + std::lock_guard lock(mutex_); + if (execution_count_ == 0) { + return std::chrono::milliseconds{0}; + } + + const auto millis = std::chrono::duration_cast(total_execution_time_); + return std::chrono::milliseconds(millis.count() / execution_count_); +} + +std::chrono::milliseconds MetricsObserver::get_total_lock_wait_time() const { + std::lock_guard lock(mutex_); + return std::chrono::duration_cast(total_lock_wait_time_); +} + +std::chrono::milliseconds MetricsObserver::get_total_execution_time() const { + std::lock_guard lock(mutex_); + return std::chrono::duration_cast(total_execution_time_); +} + +size_t MetricsObserver::get_lock_failure_count() const { + std::lock_guard lock(mutex_); + return lock_failure_count_; +} +} diff --git a/test/utils/MetricsObserver.hpp b/test/utils/MetricsObserver.hpp new file mode 100644 index 0000000..87593ad --- /dev/null +++ b/test/utils/MetricsObserver.hpp @@ -0,0 +1,28 @@ +#ifndef MATADOR_METRICS_OBSERVER_HPP +#define MATADOR_METRICS_OBSERVER_HPP + +#include "matador/utils/message_bus.hpp" + +#include "matador/sql/statement_cache.hpp" + +namespace matador::test { +class MetricsObserver { +public: + explicit MetricsObserver(utils::message_bus &bus); + + std::chrono::milliseconds get_average_lock_wait_time() const; + std::chrono::milliseconds get_average_execution_time() const; + std::chrono::milliseconds get_total_lock_wait_time() const; + std::chrono::milliseconds get_total_execution_time() const; + size_t get_lock_failure_count() const; + +private: + std::vector subscriptions; + mutable std::mutex mutex_; + size_t lock_failure_count_{0}; + size_t execution_count_{0}; + std::chrono::nanoseconds total_lock_wait_time_{0}; + std::chrono::nanoseconds total_execution_time_{0}; +}; +} +#endif //MATADOR_METRICS_OBSERVER_HPP diff --git a/test/utils/RecordingObserver.cpp b/test/utils/RecordingObserver.cpp new file mode 100644 index 0000000..5409318 --- /dev/null +++ b/test/utils/RecordingObserver.cpp @@ -0,0 +1,28 @@ +#include "RecordingObserver.hpp" + +namespace matador::test { +RecordingObserver::RecordingObserver(utils::message_bus& bus) { + subscriptions.push_back(bus.subscribe([this](const sql::statement_accessed_event &ev) { + std::lock_guard lock(mutex); + events.push(utils::message::from_ref(ev)); + })); + subscriptions.push_back(bus.subscribe([this](const sql::statement_added_event &ev) { + std::lock_guard lock(mutex); + events.push(utils::message::from_ref(ev)); + })); + subscriptions.push_back(bus.subscribe([this](const sql::statement_evicted_event &ev) { + std::lock_guard lock(mutex); + events.push(utils::message::from_ref(ev)); + })); +} + +std::optional RecordingObserver::poll() { + std::lock_guard lock(mutex); + if (events.empty()) { + return std::nullopt; + } + auto evt = events.front(); + events.pop(); + return evt; +} +} diff --git a/test/utils/RecordingObserver.hpp b/test/utils/RecordingObserver.hpp new file mode 100644 index 0000000..aeb2d81 --- /dev/null +++ b/test/utils/RecordingObserver.hpp @@ -0,0 +1,24 @@ +#ifndef MATADOR_RECORDING_OBSERVER_HPP +#define MATADOR_RECORDING_OBSERVER_HPP + +#include "matador/utils/message_bus.hpp" + +#include "matador/sql/statement_cache.hpp" + +#include + +namespace matador::test { +class RecordingObserver final { +public: + explicit RecordingObserver(utils::message_bus &bus); + + std::optional poll(); + +private: + std::vector subscriptions; + std::mutex mutex; + std::queue events; +}; +} + +#endif //MATADOR_RECORDING_OBSERVER_HPP