sascha
/
query
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

implemented the first version of statement_cache

This commit is contained in:
sascha 2025-07-30 10:34:33 +02:00
parent f90a670fb3
commit ebd8bccfb3
31 changed files with 575 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backends/postgres/include/postgres_statement.hpp
View File

@ -12,7 +12,7 @@ namespace matador::backends::postgres {
class postgres_statement final : public sql::statement_impl
{
public:
postgres_statement(PGconn *db, PGresult *result, std::string name, const sql::query_context &query);
postgres_statement(PGconn *db, std::string name, const sql::query_context &query);
utils::result<size_t, utils::error> execute() override;
utils::result<std::unique_ptr<sql::query_result_impl>, utils::error> fetch() override;
@ -22,7 +22,6 @@ protected:
private:
PGconn *db_{nullptr};
PGresult *result_{nullptr};
std::string name_;

2
backends/postgres/src/postgres_connection.cpp
View File

@ -132,7 +132,7 @@ utils::result<std::unique_ptr<sql::statement_impl>, utils::error> postgres_conne
return utils::failure(make_error(sql::error_code::PREPARE_FAILED, result, conn_, "Failed to prepare", context.sql));
}
std::unique_ptr<sql::statement_impl> s(std::make_unique<postgres_statement>(conn_, result, statement_name, context));
std::unique_ptr<sql::statement_impl> s(std::make_unique<postgres_statement>(conn_, statement_name, context));
return utils::ok(std::move(s));
}

3
backends/postgres/src/postgres_statement.cpp
View File

@ -4,10 +4,9 @@
namespace matador::backends::postgres {
postgres_statement::postgres_statement(PGconn *db, PGresult *result, std::string name, const sql::query_context &query)
postgres_statement::postgres_statement(PGconn *db, std::string name, const sql::query_context &query)
: statement_impl(query)
, db_(db)
, result_(result)
, name_(std::move(name))
, binder_(query_.bind_vars.size())
{}

4
include/matador/orm/session.hpp
View File

@ -23,7 +23,7 @@ utils::error make_error(error_code ec, const std::string &msg);
class session final {
public:
explicit session(sql::connection_pool<sql::connection> &pool);
explicit session(sql::connection_pool &pool);
template<typename Type>
[[nodiscard]] utils::result<void, utils::error> attach(const std::string &table_name) const;
@ -129,7 +129,7 @@ private:
static query::fetchable_query build_select_query(entity_query_data &&data);
private:
sql::connection_pool<sql::connection> &pool_;
sql::connection_pool &pool_;
const sql::dialect &dialect_;
std::unique_ptr<object::schema> schema_;

3
include/matador/query/intermediates/executable_query.hpp
View File

@ -8,6 +8,7 @@
namespace matador::sql {
class executor;
class prepared_executor;
class statement;
struct query_context;
}
@ -19,7 +20,7 @@ public:
using query_intermediate::query_intermediate;
[[nodiscard]] utils::result<size_t, utils::error> execute(const sql::executor &exec) const;
[[nodiscard]] utils::result<sql::statement, utils::error> prepare(const sql::executor &exec) const;
[[nodiscard]] utils::result<sql::statement, utils::error> prepare(sql::prepared_executor &exec) const;
[[nodiscard]] sql::query_context compile(const sql::executor &exec) const;
[[nodiscard]] std::string str(const sql::executor &exec) const;
};

3
include/matador/query/intermediates/fetchable_query.hpp
View File

@ -13,6 +13,7 @@
namespace matador::sql {
class executor;
class prepared_executor;
class statement;
}
@ -68,7 +69,7 @@ public:
return utils::ok(std::optional<Type>{std::nullopt});
}
[[nodiscard]] utils::result<sql::statement, utils::error> prepare(const sql::executor &exec) const;
[[nodiscard]] utils::result<sql::statement, utils::error> prepare(sql::prepared_executor &exec) const;
[[nodiscard]] std::string str(const sql::executor &exec) const;

9
include/matador/sql/abstract_sql_logger.hpp
View File

@ -1,21 +1,22 @@
#ifndef MATADOR_BASIC_SQL_LOGGER_HPP
#define MATADOR_BASIC_SQL_LOGGER_HPP
#include <memory>
#include <string>
namespace matador::sql {
/**
* @brief Base class for sql logging
* @brief Base class for SQL logging
*
* This class acts as a base class to
* implement a concrete logger for sql
* implement a concrete logger for SQL
* statements.
*
* It provides interfaces to handle
* the establishing and closing of
* a database connection as well as
* when a sql statement is about to
* when a SQL statement is about to
* execute or going to be prepared.
*/
class abstract_sql_logger
@ -94,6 +95,8 @@ public:
void on_prepare(const std::string &) override { }
};
const auto null_logger = std::make_shared<null_sql_logger>();
}
#endif //MATADOR_BASIC_SQL_LOGGER_HPP

10
include/matador/sql/connection.hpp
View File

@ -13,12 +13,10 @@
namespace matador::sql {
class connection_impl;
const auto null_logger = std::make_shared<null_sql_logger>();
/**
* @brief The connection class represents a connection to a database.
*/
class connection final : public executor {
class connection final : public executor, public prepared_executor {
public:
using logger_ptr = std::shared_ptr<abstract_sql_logger>;
/**
@ -136,14 +134,18 @@ public:
[[nodiscard]] utils::result<std::unique_ptr<query_result_impl>, utils::error> fetch(const query_context &ctx) const override;
[[nodiscard]] utils::result<size_t, utils::error> execute(const query_context &ctx) const override;
[[nodiscard]] utils::result<statement, utils::error> prepare(const query_context &ctx) const override;
[[nodiscard]] utils::result<statement, utils::error> prepare(const query_context &ctx) override;
[[nodiscard]] std::string str( const query_context& ctx ) const override;
[[nodiscard]] const class dialect &dialect() const override;
private:
[[nodiscard]] utils::result<std::unique_ptr<statement_impl>, utils::error> perform_prepare(const query_context &ctx) const;
private:
friend class fetchable_query;
friend class session;
friend class statement_cache;
connection_info connection_info_;
std::unique_ptr<connection_impl> connection_;

178
include/matador/sql/connection_pool.hpp
View File

@ -1,180 +1,73 @@
#ifndef QUERY_CONNECTION_POOL_HPP
#define QUERY_CONNECTION_POOL_HPP
#include "matador/sql/connection.hpp"
#include "matador/sql/connection_info.hpp"
#include <chrono>
#include <condition_variable>
#include <mutex>
#include <string>
#include <optional>
#include <condition_variable>
#include <thread>
#include <unordered_map>
namespace matador::sql {
template < class Connection >
class connection_pool;
template < class Connection >
using IdConnection = std::pair<size_t, Connection>;
struct identifiable_connection {
identifiable_connection(size_t id, connection conn);
size_t id{};
connection conn;
};
template < class Connection >
class connection_ptr
{
class connection_ptr {
public:
connection_ptr(IdConnection<Connection> *c, connection_pool<Connection> *pool)
: connection_(c), pool_(pool) {}
connection_ptr(identifiable_connection *c, connection_pool *pool);
~connection_ptr();
connection_ptr(const connection_ptr &) = delete;
connection_ptr& operator=(const connection_ptr &) = delete;
connection_ptr(connection_ptr &&x) noexcept
: connection_(x.connection_)
, pool_(x.pool_)
{
x.connection_ = nullptr;
x.pool_ = nullptr;
}
connection_ptr& operator=(connection_ptr &&x) noexcept
{
if (this == &x) {
return *this;
}
connection_ptr(connection_ptr &&x) noexcept;
std::swap(connection_, x.connection_);
std::swap(pool_, x.pool_);
connection_ptr& operator=(connection_ptr &&x) noexcept;
return *this;
}
connection* operator->() const;
connection& operator*() const;
Connection* operator->() { return &connection_->second; }
Connection& operator*() { return connection_->second; }
[[nodiscard]] std::optional<size_t> id() const
{
if (connection_) {
return connection_->first;
} else {
return std::nullopt;
}
}
[[nodiscard]] bool valid() const { return connection_ != nullptr; }
[[nodiscard]] std::optional<size_t> id() const;
[[nodiscard]] bool valid() const;
private:
friend class connection_pool<Connection>;
friend class connection_pool;
IdConnection<Connection> *connection_{};
connection_pool<Connection> *pool_{};
identifiable_connection *connection_{};
connection_pool *pool_{};
};
template < class Connection >
class connection_pool
{
class connection_pool {
public:
using connection_pointer = connection_ptr<Connection>;
connection_pool(const std::string &dns, size_t count);
public:
connection_pool(const std::string &dns, size_t count)
: info_(connection_info::parse(dns)) {
connection_repo_.reserve(count);
while (count) {
connection_repo_.emplace_back(count, info_);
auto &conn = connection_repo_.back();
idle_connections_.emplace(conn.first, &conn);
// Todo: handle result
const auto result = conn.second.open();
if (!result) {
throw std::runtime_error("Failed to open connection");
}
--count;
}
}
connection_ptr acquire();
connection_ptr try_acquire();
connection_ptr acquire(size_t id);
connection_pointer acquire() {
std::unique_lock<std::mutex> lock(mutex_);
while (idle_connections_.empty()) {
cv.wait(lock);
}
void release(identifiable_connection *c);
void release(connection_ptr &c);
return get_next_connection();
}
std::size_t size() const;
std::size_t idle() const;
std::size_t inuse() const;
connection_pointer try_acquire() {
std::unique_lock<std::mutex> lock(mutex_);
if (idle_connections_.empty()) {
return {nullptr, this};
}
return get_next_connection();
}
connection_pointer acquire(size_t id) {
using namespace std::chrono_literals;
pointer next_connection{nullptr};
auto try_count{0};
std::unique_lock<std::mutex> lock(mutex_);
do {
if (auto it = idle_connections_.find(id); it != idle_connections_.end()) {
next_connection = it->second;
auto node = idle_connections_.extract(it);
inuse_connections_.insert(std::move(node));
} else {
lock.unlock();
std::this_thread::sleep_for(100ms);
lock.lock();
}
} while(try_count++ < 5);
return {next_connection, this};
}
void release(IdConnection<Connection> *c) {
if (c == nullptr) {
return;
}
std::unique_lock<std::mutex> lock(mutex_);
if (auto it = inuse_connections_.find(c->first); it != inuse_connections_.end()) {
auto node = inuse_connections_.extract(it);
idle_connections_.insert(std::move(node));
}
}
void release(connection_ptr<Connection> &c) {
release(c.connection_);
c.connection_ = nullptr;
}
std::size_t size() const { return connection_repo_.size(); }
std::size_t idle() const {
std::lock_guard<std::mutex> guard(mutex_);
return idle_connections_.size();
}
std::size_t inuse() const {
std::lock_guard<std::mutex> guard(mutex_);
return inuse_connections_.size();
}
const connection_info &info() const {
return info_;
}
const connection_info &info() const;
private:
connection_pointer get_next_connection() {
pointer next_connection{nullptr};
for (auto &item : idle_connections_) {
next_connection = item.second;
auto node = idle_connections_.extract(item.first);
inuse_connections_.insert(std::move(node));
break;
}
return {next_connection, this};
}
connection_ptr get_next_connection();
private:
mutable std::mutex mutex_;
std::condition_variable cv;
std::vector<IdConnection<Connection>> connection_repo_;
using pointer = IdConnection<Connection>*;
std::vector<identifiable_connection> connection_repo_;
using pointer = identifiable_connection*;
using connection_map = std::unordered_map<size_t, pointer>;
connection_map inuse_connections_;
connection_map idle_connections_;
@ -182,11 +75,6 @@ private:
const connection_info info_;
};
template<class Connection>
connection_ptr<Connection>::~connection_ptr() {
pool_->release(connection_);
}
}
#endif //QUERY_CONNECTION_POOL_HPP

16
include/matador/sql/executor.hpp
View File

@ -12,14 +12,22 @@ struct query_context;
class query_result_impl;
class statement;
class executor {
class dialect_executor_mixin {
public:
virtual ~executor();
virtual ~dialect_executor_mixin() = default;
[[nodiscard]] virtual const class dialect& dialect() const = 0;
};
class prepared_executor : public dialect_executor_mixin {
public:
[[nodiscard]] virtual utils::result<statement, utils::error> prepare(const query_context &ctx) = 0;
};
class executor : public dialect_executor_mixin {
public:
[[nodiscard]] virtual utils::result<size_t, utils::error> execute(const query_context &ctx) const = 0;
[[nodiscard]] virtual utils::result<std::unique_ptr<query_result_impl>, utils::error> fetch(const query_context &ctx) const = 0;
[[nodiscard]] virtual utils::result<statement, utils::error> prepare(const query_context &ctx) const = 0;
[[nodiscard]] virtual std::string str(const query_context &ctx) const = 0;
};

37
include/matador/sql/interface/statement_proxy.hpp Normal file
View File

@ -0,0 +1,37 @@
#ifndef STATEMENT_PROXY_HPP
#define STATEMENT_PROXY_HPP
#include "matador/sql/interface/statement_impl.hpp"
namespace matador::sql {
class statement_proxy {
protected:
explicit statement_proxy(std::unique_ptr<statement_impl>&& stmt);
public:
virtual ~statement_proxy() = default;
virtual utils::result<size_t, utils::error> execute() = 0;
virtual utils::result<std::unique_ptr<query_result_impl>, utils::error> fetch() = 0;
template<class Type>
void bind(const Type &obj) {
statement_->bind_object(obj);
}
template<typename Type>
void bind(size_t pos, Type &value) {
statement_->bind(pos, value);
}
void bind(size_t pos, const char *value, size_t size) const;
void bind(size_t pos, std::string &val, size_t size) const;
void reset() const;
protected:
std::unique_ptr<statement_impl> statement_;
};
}
#endif //STATEMENT_PROXY_HPP

49
include/matador/sql/statement.hpp
View File

@ -4,7 +4,7 @@
#include "matador/sql/abstract_sql_logger.hpp"
#include "matador/sql/query_result.hpp"
#include "matador/sql/interface/statement_impl.hpp"
#include "matador/sql/interface/statement_proxy.hpp"
#include "matador/utils/error.hpp"
#include "matador/utils/result.hpp"
@ -18,26 +18,28 @@ class identifier_binder;
}
class statement_impl;
class statement_proxy;
class statement {
public:
using logger_ptr = std::shared_ptr<abstract_sql_logger>;
/**
* Creates a statement initialized from the
* given statement implementation object holding
* the implementation for the selected database
*
* @param impl The statement implementation object
* @param logger The logger handler to write sql log messages to
* @param proxy The statement proxy implementation object
* @param logger The logger handler to write SQL log messages to
*/
explicit statement(std::unique_ptr<statement_impl> impl,
const std::shared_ptr<abstract_sql_logger> &logger = std::make_shared<null_sql_logger>());
explicit statement(const std::shared_ptr<statement_proxy>& proxy, logger_ptr logger = null_logger);
/**
* Copy move constructor for statement
*
* @param x The statement to move from
*/
statement(statement &&x) noexcept
: statement_(std::move(x.statement_))
: statement_proxy_(std::move(x.statement_proxy_))
, logger_(std::move(x.logger_)) {
}
@ -48,11 +50,14 @@ public:
* @return Reference to this
*/
statement &operator=(statement &&x) noexcept {
statement_ = std::move(x.statement_);
statement_proxy_ = std::move(x.statement_proxy_);
logger_ = std::move(x.logger_);
return *this;
}
statement(const statement &x) = default;
statement &operator=(const statement &x) = default;
statement &bind(size_t pos, const char *value);
statement &bind(size_t pos, std::string &val, size_t size);
/**
@ -77,8 +82,8 @@ public:
/**
* Fetches the result of the prepared
* statement. If prepared statement was not
* a SELECT statement an empty query result set
* statement. If a prepared statement was not
* a SELECT statement, an empty query result set
* is returned.
*
* @tparam Type Type of the fetched result
@ -88,10 +93,10 @@ public:
utils::result<query_result<Type>, utils::error> fetch();
/**
* Fetches the result of the prepared
* statement. The type is record representing an
* statement. The type is the record representing an
* unknown variable type.
* If prepared statement was not
* a SELECT statement an empty query result set
* If the prepared statement was not
* a SELECT statement, an empty query result set
* is returned.
*
* @return The query result set
@ -99,8 +104,8 @@ public:
[[nodiscard]] utils::result<query_result<record>, utils::error> fetch() const;
/**
* Fetches the first result of a prepared statement.
* If prepared statement is empty or not
* a SELECT statement a nullptr is returned.
* If a prepared statement is empty or not
* a SELECT statement, a nullptr is returned.
*
* @tparam Type Type of the fetched result
* @return The query result set
@ -109,9 +114,9 @@ public:
utils::result<std::unique_ptr<Type>, utils::error> fetch_one();
/**
* Fetches the first result of a prepared statement.
* The type is record representing an unknown variable type.
* If prepared statement is empty or not
* a SELECT statement a nullptr is returned.
* The type is a record representing an unknown variable type.
* If a prepared statement is empty or not
* a SELECT statement, a nullptr is returned.
*
* @return The query result set
*/
@ -128,25 +133,25 @@ private:
friend class detail::identifier_binder;
private:
std::unique_ptr<statement_impl> statement_;
std::shared_ptr<statement_proxy> statement_proxy_;
std::shared_ptr<abstract_sql_logger> logger_;
};
template<typename Type>
statement &statement::bind(size_t pos, Type &value) {
statement_->bind(pos, value);
statement_proxy_->bind(pos, value);
return *this;
}
template<class Type>
statement &statement::bind(const Type &obj) {
statement_->bind_object(obj);
statement_proxy_->bind(obj);
return *this;
}
template<class Type>
utils::result<query_result<Type>, utils::error> statement::fetch() {
return statement_->fetch().and_then([](std::unique_ptr<query_result_impl> &&value) {
return statement_proxy_->fetch().and_then([](std::unique_ptr<query_result_impl> &&value) {
return utils::ok(query_result<Type>(std::forward<decltype(value)>(value)));
});
// if (!result.is_ok()) {
@ -157,7 +162,7 @@ utils::result<query_result<Type>, utils::error> statement::fetch() {
template<class Type>
utils::result<std::unique_ptr<Type>, utils::error> statement::fetch_one() {
auto result = statement_->fetch();
auto result = statement_proxy_->fetch();
if (!result.is_ok()) {
return utils::failure(result.err());
}

38
include/matador/sql/statement_cache.hpp Normal file
View File

@ -0,0 +1,38 @@
#ifndef STATEMENT_CACHE_HPP
#define STATEMENT_CACHE_HPP
#include "matador/sql/executor.hpp"
#include "matador/sql/interface/statement_proxy.hpp"
#include <list>
#include <mutex>
#include <unordered_map>
namespace matador::sql {
class connection_pool;
class statement_cache final {
public:
explicit statement_cache(connection_pool &pool, size_t max_size = 50);
[[nodiscard]] utils::result<statement, utils::error> acquire(const query_context &ctx);
[[nodiscard]] size_t size() const;
[[nodiscard]] size_t capacity() const;
[[nodiscard]] bool empty() const;
private:
using list_iterator = std::list<size_t>::iterator;
size_t max_size_{};
std::list<size_t> usage_list_; // LRU: front = most recent, back = least recent
std::unordered_map<size_t, std::pair<statement, list_iterator>> cache_map_;
std::mutex mutex_;
connection_pool &pool_;
const sql::dialect &dialect_;
};
}
#endif //STATEMENT_CACHE_HPP

1
source/core/CMakeLists.txt
View File

@ -101,6 +101,7 @@ add_library(matador-core STATIC
utils/uuid.cpp
utils/value.cpp
utils/version.cpp
../../include/matador/sql/statement_cache.hpp
)
target_link_libraries(matador-core ${CMAKE_DL_LIBS})

22
source/orm/CMakeLists.txt
View File

@ -1,4 +1,8 @@
add_library(matador-orm STATIC
../../include/matador/orm/error_code.hpp
../../include/matador/orm/session.hpp
../../include/matador/orm/session_insert_builder.hpp
../../include/matador/orm/session_query_builder.hpp
../../include/matador/query/attribute_string_writer.hpp
../../include/matador/query/basic_condition.hpp
../../include/matador/query/condition.hpp
@ -39,10 +43,6 @@ add_library(matador-orm STATIC
../../include/matador/query/query_intermediates.hpp
../../include/matador/query/query_part.hpp
../../include/matador/query/value_extractor.hpp
../../include/matador/orm/error_code.hpp
../../include/matador/orm/session.hpp
../../include/matador/orm/session_insert_builder.hpp
../../include/matador/orm/session_query_builder.hpp
../../include/matador/sql/abstract_sql_logger.hpp
../../include/matador/sql/backend_provider.hpp
../../include/matador/sql/column.hpp
@ -59,6 +59,7 @@ add_library(matador-orm STATIC
../../include/matador/sql/interface/parameter_binder.hpp
../../include/matador/sql/interface/query_result_reader.hpp
../../include/matador/sql/interface/statement_impl.hpp
../../include/matador/sql/interface/statement_proxy.hpp
../../include/matador/sql/internal/object_result_binder.hpp
../../include/matador/sql/internal/query_result_impl.hpp
../../include/matador/sql/internal/query_result_pk_resolver.hpp
@ -68,6 +69,10 @@ add_library(matador-orm STATIC
../../include/matador/sql/record.hpp
../../include/matador/sql/statement.hpp
../../include/matador/sql/table.hpp
orm/error_code.cpp
orm/session.cpp
orm/session_insert_builder.cpp
orm/session_query_builder.cpp
query/attribute_string_writer.cpp
query/basic_condition.cpp
query/condition.cpp
@ -105,30 +110,29 @@ add_library(matador-orm STATIC
query/query_part.cpp
query/query_update_intermediate.cpp
query/value_extractor.cpp
orm/error_code.cpp
orm/session.cpp
orm/session_insert_builder.cpp
orm/session_query_builder.cpp
sql/backend_provider.cpp
sql/column.cpp
sql/column_generator.cpp
sql/connection.cpp
sql/connection_info.cpp
sql/connection_pool.cpp
sql/dialect.cpp
sql/dialect_builder.cpp
sql/error_code.cpp
sql/executor.cpp
sql/field.cpp
sql/interface/connection_impl.cpp
sql/interface/query_result_reader.cpp
sql/interface/statement_impl.cpp
sql/interface/statement_proxy.cpp
sql/internal/object_result_binder.cpp
sql/internal/query_result_pk_resolver.cpp
sql/object_parameter_binder.cpp
sql/query_result.cpp
sql/record.cpp
sql/statement.cpp
sql/statement_cache.cpp
sql/table.cpp
sql/error_code.cpp
)
target_include_directories(matador-orm

8
source/orm/orm/session.cpp
View File

@ -12,10 +12,10 @@ utils::error make_error(const error_code ec, const std::string &msg) {
return utils::error(ec, msg);
}
session::session(sql::connection_pool<sql::connection> &pool)
: pool_(pool)
, dialect_(sql::backend_provider::instance().connection_dialect(pool_.info().type))
, schema_(std::make_unique<object::schema>(dialect_.default_schema_name())) {
session::session(sql::connection_pool &pool)
: pool_(pool)
, dialect_(sql::backend_provider::instance().connection_dialect(pool_.info().type))
, schema_(std::make_unique<object::schema>(dialect_.default_schema_name())) {
}
utils::result<void, utils::error> session::create_schema() const {

2
source/orm/query/intermediates/executable_query.cpp
View File

@ -12,7 +12,7 @@ utils::result<size_t, utils::error> executable_query::execute(const sql::executo
return exec.execute(compiler.compile(*context_, exec.dialect(), std::nullopt));
}
utils::result<sql::statement, utils::error> executable_query::prepare(const sql::executor &exec) const {
utils::result<sql::statement, utils::error> executable_query::prepare(sql::prepared_executor &exec) const {
query_compiler compiler;
context_->mode = query_mode::Prepared;
return exec.prepare(compiler.compile(*context_, exec.dialect(), std::nullopt));

2
source/orm/query/intermediates/fetchable_query.cpp
View File

@ -43,7 +43,7 @@ utils::result<std::unique_ptr<sql::query_result_impl>, utils::error> fetchable_q
return exec.fetch(compiler.compile(*context_, exec.dialect(), std::nullopt));
}
utils::result<sql::statement, utils::error> fetchable_query::prepare(const sql::executor &exec) const {
utils::result<sql::statement, utils::error> fetchable_query::prepare(sql::prepared_executor &exec) const {
query_compiler compiler;
context_->mode = query_mode::Prepared;
return exec.prepare(compiler.compile(*context_, exec.dialect(), std::nullopt));

78
source/orm/sql/connection.cpp
View File

@ -13,6 +13,21 @@
namespace matador::sql {
namespace internal {
class connection_statement_proxy final : public statement_proxy {
public:
explicit connection_statement_proxy(std::unique_ptr<statement_impl>&& stmt)
: statement_proxy(std::move(stmt)) {}
utils::result<size_t, utils::error> execute() override {
return statement_->execute();
}
utils::result<std::unique_ptr<query_result_impl>, utils::error> fetch() override {
return statement_->fetch();
}
};
}
connection::connection(connection_info info, const std::shared_ptr<abstract_sql_logger> &sql_logger)
: connection_info_(std::move(info))
, logger_(sql_logger)
@ -57,8 +72,10 @@ connection & connection::operator=(connection &&x) noexcept {
return *this;
}
connection::~connection()
{
connection::~connection() {
if (!connection_) {
return;
}
if (connection_->is_open()) {
connection_->close();
}
@ -66,8 +83,7 @@ connection::~connection()
connection_ = nullptr;
}
utils::result<void, utils::error> connection::open() const
{
utils::result<void, utils::error> connection::open() const {
const auto res = is_open();
if (res.is_error()) {
return utils::failure(res.err());
@ -79,8 +95,7 @@ utils::result<void, utils::error> connection::open() const
return utils::ok<void>();
}
utils::result<void, utils::error> connection::close() const
{
utils::result<void, utils::error> connection::close() const {
logger_->on_close();
return connection_->close();
}
@ -185,27 +200,17 @@ utils::result<size_t, utils::error> connection::execute(const query_context& ctx
return execute(ctx.sql);
}
utils::result<statement, utils::error> connection::prepare(const query_context &ctx) const
{
if (ctx.command != sql_command::SQL_CREATE_TABLE && (ctx.prototype.empty() || has_unknown_columns(ctx.prototype))) {
if (const auto result = describe(ctx.table.name); result.is_ok()) {
for (auto &col: ctx.prototype) {
const auto rit = std::find_if(std::begin(*result), std::end(*result),
[&col](const auto &value) {
return value.name() == col.name();
});
if (col.type() == utils::basic_type::type_null && rit != result->end()) {
const_cast<object::attribute_definition&>(col).type(rit->type());
}
}
}
utils::result<statement, utils::error> connection::prepare(const query_context &ctx) {
auto result = perform_prepare(ctx);
if (!result) {
return utils::failure(result.err());
}
if (auto result = connection_->prepare(ctx); result.is_ok()) {
return utils::ok(statement(result.release()));
}
return utils::ok(statement(std::unique_ptr<statement_impl>{}));
return utils::ok(statement(
std::make_shared<internal::connection_statement_proxy>(result.release()),
logger_)
);
}
std::string connection::str( const query_context& ctx ) const
@ -218,4 +223,27 @@ const class dialect &connection::dialect() const
return connection_->dialect();
}
utils::result<std::unique_ptr<statement_impl>, utils::error> connection::perform_prepare(const query_context& ctx) const {
if (ctx.command != sql_command::SQL_CREATE_TABLE && (ctx.prototype.empty() || has_unknown_columns(ctx.prototype))) {
if (const auto result = describe(ctx.table.name); result.is_ok()) {
for (auto &col: ctx.prototype) {
const auto rit = std::find_if(
std::begin(*result),
std::end(*result),
[&col](const auto &value) { return value.name() == col.name(); }
);
if (col.type() == utils::basic_type::type_null && rit != result->end()) {
const_cast<object::attribute_definition&>(col).type(rit->type());
}
}
}
}
auto result = connection_->prepare(ctx);
if (!result) {
return utils::failure(result.err());
}
return utils::ok(result.release());
}
}

155
source/orm/sql/connection_pool.cpp Normal file
View File

@ -0,0 +1,155 @@
#include "matador/sql/connection_pool.hpp"
#include <chrono>
#include <thread>
namespace matador::sql {
identifiable_connection::identifiable_connection(const size_t id, connection conn)
: id(id)
, conn(std::move(conn)){}
connection_ptr::~connection_ptr() {
pool_->release(connection_);
}
connection_ptr::connection_ptr(identifiable_connection* c, connection_pool* pool)
: connection_(c)
, pool_(pool) {}
connection_ptr::connection_ptr(connection_ptr&& x) noexcept
: connection_(x.connection_)
, pool_(x.pool_) {
x.connection_ = nullptr;
x.pool_ = nullptr;
}
connection_ptr& connection_ptr::operator=(connection_ptr&& x) noexcept {
if (this == &x) {
return *this;
}
std::swap(connection_, x.connection_);
std::swap(pool_, x.pool_);
return *this;
}
connection* connection_ptr::operator->() const {
return &connection_->conn;
}
connection& connection_ptr::operator*() const {
return connection_->conn;
}
std::optional<size_t> connection_ptr::id() const {
if (connection_) {
return connection_->id;
}
return std::nullopt;
}
bool connection_ptr::valid() const {
return connection_ != nullptr;
}
connection_pool::connection_pool(const std::string& dns, size_t count)
: info_(connection_info::parse(dns)) {
connection_repo_.reserve(count);
while (count) {
connection_repo_.emplace_back(count, connection{info_});
auto &conn = connection_repo_.back();
idle_connections_.emplace(conn.id, &conn);
// Todo: handle result
const auto result = conn.conn.open();
if (!result) {
throw std::runtime_error("Failed to open connection");
}
--count;
}
}
connection_ptr connection_pool::acquire() {
std::unique_lock lock(mutex_);
while (idle_connections_.empty()) {
cv.wait(lock);
}
return get_next_connection();
}
connection_ptr connection_pool::try_acquire() {
std::unique_lock lock(mutex_);
if (idle_connections_.empty()) {
return {nullptr, this};
}
return get_next_connection();
}
connection_ptr connection_pool::acquire(const size_t id) {
using namespace std::chrono_literals;
pointer next_connection{nullptr};
auto try_count{0};
std::unique_lock lock(mutex_);
do {
if (auto it = idle_connections_.find(id); it != idle_connections_.end()) {
next_connection = it->second;
auto node = idle_connections_.extract(it);
inuse_connections_.insert(std::move(node));
} else {
lock.unlock();
std::this_thread::sleep_for(100ms);
lock.lock();
}
} while(try_count++ < 5);
return {next_connection, this};
}
void connection_pool::release(identifiable_connection* c) {
if (c == nullptr) {
return;
}
std::unique_lock lock(mutex_);
if (const auto it = inuse_connections_.find(c->id); it != inuse_connections_.end()) {
auto node = inuse_connections_.extract(it);
idle_connections_.insert(std::move(node));
}
}
void connection_pool::release(connection_ptr& c) {
release(c.connection_);
c.connection_ = nullptr;
}
std::size_t connection_pool::size() const {
return connection_repo_.size();
}
std::size_t connection_pool::idle() const {
std::lock_guard guard(mutex_);
return idle_connections_.size();
}
std::size_t connection_pool::inuse() const {
std::lock_guard guard(mutex_);
return inuse_connections_.size();
}
const connection_info& connection_pool::info() const {
return info_;
}
connection_ptr connection_pool::get_next_connection() {
pointer next_connection{nullptr};
for (auto & [id, conn] : idle_connections_) {
next_connection = conn;
auto node = idle_connections_.extract(id);
inuse_connections_.insert(std::move(node));
break;
}
return {next_connection, this};
}
}

1
source/orm/sql/executor.cpp
View File

@ -1,5 +1,4 @@
#include "matador/sql/executor.hpp"
namespace matador::sql {
executor::~executor() = default;
}

17
source/orm/sql/interface/statement_proxy.cpp Normal file
View File

@ -0,0 +1,17 @@
#include "matador/sql/interface/statement_proxy.hpp"
namespace matador::sql {
statement_proxy::statement_proxy(std::unique_ptr<statement_impl>&& stmt)
: statement_(std::move(stmt)){}
void statement_proxy::bind(const size_t pos, const char* value, const size_t size) const {
statement_->bind(pos, value, size);
}
void statement_proxy::bind(const size_t pos, std::string& val, const size_t size) const {
statement_->bind(pos, val, size);
}
void statement_proxy::reset() const {
statement_->reset();
}
}

30
source/orm/sql/statement.cpp
View File

@ -3,30 +3,28 @@
#include "matador/sql/field.hpp"
#include <algorithm>
#include <utility>
namespace matador::sql {
statement::statement(std::unique_ptr<statement_impl> impl, const std::shared_ptr<abstract_sql_logger> &logger)
: statement_(std::move(impl))
, logger_(logger)
{}
statement::statement(const std::shared_ptr<statement_proxy>& proxy, logger_ptr logger)
: statement_proxy_(proxy)
, logger_(std::move(logger)){}
statement &statement::bind(const size_t pos, const char *value)
{
statement_->bind(pos, value, 0);
statement &statement::bind(const size_t pos, const char *value) {
statement_proxy_->bind(pos, value, strlen(value));
return *this;
}
statement &statement::bind(const size_t pos, std::string &val, const size_t size)
{
statement_->bind(pos, val, size);
statement_proxy_->bind(pos, val, size);
return *this;
}
utils::result<size_t, utils::error> statement::execute() const
{
utils::result<size_t, utils::error> statement::execute() const {
// logger_.info(statement_->query_.sql);
return statement_->execute();
return statement_proxy_->execute();
}
//bool is_unknown(const std::vector<object::column_definition> &columns) {
@ -35,13 +33,13 @@ utils::result<size_t, utils::error> statement::execute() const
// });
//}
utils::result<query_result<record>, utils::error> statement::fetch() const
{
utils::result<query_result<record>, utils::error> statement::fetch() const {
// if (is_unknown(statement_->query_.prototype)) {
//
// }
auto result = statement_->fetch();
auto result = statement_proxy_->fetch();
if (!result.is_ok()) {
return utils::failure(result.err());
}
@ -51,7 +49,7 @@ utils::result<query_result<record>, utils::error> statement::fetch() const
utils::result<std::optional<record>, utils::error> statement::fetch_one() const {
// logger_.info(statement_->query_.sql);
auto result = statement_->fetch();
auto result = statement_proxy_->fetch();
if (!result.is_ok()) {
return utils::failure(result.err());
}
@ -67,7 +65,7 @@ utils::result<std::optional<record>, utils::error> statement::fetch_one() const
void statement::reset() const
{
statement_->reset();
statement_proxy_->reset();
}
}

72
source/orm/sql/statement_cache.cpp Normal file
View File

@ -0,0 +1,72 @@
#include "matador/sql/statement_cache.hpp"
#include "matador/sql/backend_provider.hpp"
#include "matador/sql/error_code.hpp"
#include "matador/sql/connection_pool.hpp"
namespace matador::sql {
namespace internal {
class statement_cache_proxy final : public statement_proxy {
public:
explicit statement_cache_proxy(std::unique_ptr<statement_impl>&& stmt)
: statement_proxy(std::move(stmt)) {}
utils::result<size_t, utils::error> execute() override {
return statement_->execute();
}
utils::result<std::unique_ptr<query_result_impl>, utils::error> fetch() override {
return statement_->fetch();
}
};
}
statement_cache::statement_cache(connection_pool &pool, const size_t max_size)
: max_size_(max_size)
, pool_(pool)
, dialect_(backend_provider::instance().connection_dialect(pool_.info().type)) {}
utils::result<statement, utils::error> statement_cache::acquire(const query_context& ctx) {
std::unique_lock lock(mutex_);
// hash statement
const auto key = std::hash<std::string>{}(ctx.sql);
// Found in cache. Move it to of the LRU list
if (const auto it = cache_map_.find(key); it != cache_map_.end()) {
usage_list_.splice(usage_list_.begin(), usage_list_, it->second.second);
return utils::ok(it->second.first);
}
// Prepare a new statement
// acquire pool connection
const auto conn = pool_.acquire();
auto result = conn->perform_prepare(ctx);
if (!result) {
return utils::failure(utils::error{error_code::PREPARE_FAILED, std::string("Failed to prepare")});
}
// If cache max size reached ensure space
if (cache_map_.size() >= max_size_) {
const size_t& lru_key = usage_list_.back();
cache_map_.erase(lru_key);
usage_list_.pop_back();
}
usage_list_.push_front(key);
const auto it = cache_map_.insert({
key,
std::make_pair(statement{
std::make_shared<internal::statement_cache_proxy>(result.release())},
usage_list_.begin())
}).first;
return utils::ok(it->second.first);
}
size_t statement_cache::size() const {
return cache_map_.size();
}
size_t statement_cache::capacity() const {
return max_size_;
}
bool statement_cache::empty() const {
return cache_map_.empty();
}
}

2
test/backends/SessionFixture.hpp
View File

@ -15,7 +15,7 @@ public:
~SessionFixture();
protected:
sql::connection_pool<sql::connection> pool;
sql::connection_pool pool;
std::stack <std::string> tables_to_drop;
orm::session ses;

2
test/backends/StatementCacheTest.cpp
View File

@ -20,7 +20,7 @@ public:
~StatementCacheFixture() = default;
protected:
sql::connection_pool<sql::connection> pool;
sql::connection_pool pool;
orm::session ses;
};

2
test/orm/CMakeLists.txt
View File

@ -26,6 +26,8 @@ add_executable(OrmTests
sql/FieldTest.cpp
utils/auto_reset_event.cpp
utils/auto_reset_event.hpp
sql/StatementCacheTest.cpp
sql/ConnectionPoolFixture.hpp
)
target_link_libraries(OrmTests matador-orm matador-core Catch2::Catch2WithMain)

3
test/orm/backend/test_backend_service.cpp
View File

@ -7,8 +7,7 @@
namespace matador::test::orm {
sql::connection_impl *test_backend_service::create(const sql::connection_info &info)
{
sql::connection_impl *test_backend_service::create(const sql::connection_info &info) {
return noop_connections_.insert(std::make_unique<test_connection>(info)).first->get();
}

27
test/orm/sql/ConnectionPoolFixture.hpp Normal file
View File

@ -0,0 +1,27 @@
#ifndef CONNECTION_POOL_FIXTURE_HPP
#define CONNECTION_POOL_FIXTURE_HPP
#include "matador/sql/backend_provider.hpp"
#include "matador/sql/connection.hpp"
#include "matador/sql/interface/connection_impl.hpp"
#include "../backend/test_backend_service.hpp"
namespace matador::test::orm {
class ConnectionPoolFixture {
public:
ConnectionPoolFixture() {
sql::backend_provider::instance().register_backend("noop", std::make_unique<test_backend_service>());
db = std::make_unique<sql::connection>("noop://noop.db");
}
~ConnectionPoolFixture() = default;
protected:
std::unique_ptr<sql::connection> db;
};
}
#endif //CONNECTION_POOL_FIXTURE_HPP

38
test/orm/sql/ConnectionPoolTest.cpp
View File

@ -1,39 +1,21 @@
#include <catch2/catch_test_macros.hpp>
#include "matador/sql/backend_provider.hpp"
#include "matador/sql/connection.hpp"
#include "matador/sql/connection_pool.hpp"
#include "../backend/test_connection.hpp"
#include "../backend/test_backend_service.hpp"
#include "../utils/auto_reset_event.hpp"
#include "ConnectionPoolFixture.hpp"
#include <thread>
using namespace matador::sql;
using namespace matador::test::utils;
using namespace matador::test::orm;
namespace matador::test::orm {
class ConnectionPoolFixture {
public:
ConnectionPoolFixture() {
backend_provider::instance().register_backend("noop", std::make_unique<test_backend_service>());
db = std::make_unique<sql::connection>("noop://noop.db");
}
~ConnectionPoolFixture() = default;
protected:
std::unique_ptr<connection> db;
};
}
TEST_CASE_METHOD(ConnectionPoolFixture, "Create connection pool", "[connection pool]") {
using pool_t = connection_pool<test_connection>;
pool_t pool("noop://noop.db", 4);
connection_pool pool("noop://noop.db", 4);
REQUIRE(pool.size() == 4);
REQUIRE(pool.idle() == 4);
@ -75,9 +57,7 @@ TEST_CASE_METHOD(ConnectionPoolFixture, "Create connection pool", "[connection p
}
TEST_CASE_METHOD(ConnectionPoolFixture, "Acquire connection by id", "[connection pool]") {
using pool_t = connection_pool<test_connection>;
pool_t pool("noop://noop.db", 4);
connection_pool pool("noop://noop.db", 4);
REQUIRE(pool.size() == 4);
REQUIRE(pool.idle() == 4);
@ -105,9 +85,7 @@ TEST_CASE_METHOD(ConnectionPoolFixture, "Acquire connection by id", "[connection
}
TEST_CASE("Try acquire connection", "[connection pool][try acquire]") {
using pool_t = connection_pool<test_connection>;
pool_t pool("noop://noop.db", 1);
connection_pool pool("noop://noop.db", 1);
REQUIRE(pool.size() == 1);
REQUIRE(pool.idle() == 1);
@ -145,7 +123,7 @@ TEST_CASE("Try acquire connection", "[connection pool][try acquire]") {
auto_reset_event reset_main_event;
auto_reset_event reset_thread_event;
std::thread t([&reset_main_event, &reset_thread_event, &pool]() {
std::thread t([&reset_main_event, &reset_thread_event, &pool] {
auto c1 = pool.acquire();
REQUIRE(c1.valid());
REQUIRE(c1.id());

36
test/orm/sql/StatementCacheTest.cpp Normal file
View File

@ -0,0 +1,36 @@
#include <catch2/catch_test_macros.hpp>
#include <matador/query/query.hpp>
#include "matador/sql/connection_pool.hpp"
#include "matador/sql/statement_cache.hpp"
#include "../backend/test_backend_service.hpp"
#include "ConnectionPoolFixture.hpp"
using namespace matador::test;
using namespace matador::sql;
using namespace matador::query;
TEST_CASE("Test statement cache", "[statement][cache]") {
backend_provider::instance().register_backend("noop", std::make_unique<orm::test_backend_service>());
connection_pool pool("noop://noop.db", 4);
statement_cache cache(pool, 4);
query_context ctx;
ctx.sql = "SELECT * FROM person";
REQUIRE(cache.capacity() == 4);
REQUIRE(cache.empty());
auto result = cache.acquire(ctx);
REQUIRE(result);
REQUIRE(cache.size() == 1);
REQUIRE(!cache.empty());
REQUIRE(cache.capacity() == 4);
const auto stmt = result.value();
}