SQLBackend: Use std::optional return value instead of exceptions

For wrong value type when retrieving a value from the SQL results row.

Profiling showed that most of the SQL load time was spent in handling
these exceptions, and using std::optional instead produced a > 11x
speedup (10 seconds vs. 115 seconds) when loading a large file.
This commit is contained in:
John Ralls 2023-08-10 13:36:39 -07:00
parent beec420486
commit 5781f3445b
11 changed files with 182 additions and 217 deletions

View File

@ -29,6 +29,7 @@
#include <dbi/dbi-dev.h>
#include <cmath>
#include <gnc-datetime.hpp>
#include <sys/_types/_timeval.h>
#include "gnc-dbisqlresult.hpp"
#include "gnc-dbisqlconnection.hpp"
@ -98,16 +99,16 @@ GncDbiSqlResult::IteratorImpl::operator++()
return m_inst->m_sentinel;
}
int64_t
std::optional<int64_t>
GncDbiSqlResult::IteratorImpl::get_int_at_col(const char* col) const
{
auto type = dbi_result_get_field_type (m_inst->m_dbi_result, col);
if(type != DBI_TYPE_INTEGER)
throw (std::invalid_argument{"Requested integer from non-integer column."});
return dbi_result_get_longlong (m_inst->m_dbi_result, col);
return std::nullopt;
return std::optional<int64_t>{dbi_result_get_longlong (m_inst->m_dbi_result, col)};
}
double
std::optional<double>
GncDbiSqlResult::IteratorImpl::get_float_at_col(const char* col) const
{
constexpr double float_precision = 1000000.0;
@ -115,56 +116,52 @@ GncDbiSqlResult::IteratorImpl::get_float_at_col(const char* col) const
auto attrs = dbi_result_get_field_attribs (m_inst->m_dbi_result, col);
if(type != DBI_TYPE_DECIMAL ||
(attrs & DBI_DECIMAL_SIZEMASK) != DBI_DECIMAL_SIZE4)
throw (std::invalid_argument{"Requested float from non-float column."});
return std::nullopt;
auto locale = gnc_push_locale (LC_NUMERIC, "C");
auto interim = dbi_result_get_float(m_inst->m_dbi_result, col);
gnc_pop_locale (LC_NUMERIC, locale);
double retval = static_cast<double>(round(interim * float_precision)) / float_precision;
return retval;
return std::optional<double>{retval};
}
double
std::optional<double>
GncDbiSqlResult::IteratorImpl::get_double_at_col(const char* col) const
{
auto type = dbi_result_get_field_type (m_inst->m_dbi_result, col);
auto attrs = dbi_result_get_field_attribs (m_inst->m_dbi_result, col);
if(type != DBI_TYPE_DECIMAL ||
(attrs & DBI_DECIMAL_SIZEMASK) != DBI_DECIMAL_SIZE8)
throw (std::invalid_argument{"Requested double from non-double column."});
return std::nullopt;
auto locale = gnc_push_locale (LC_NUMERIC, "C");
auto retval = dbi_result_get_double(m_inst->m_dbi_result, col);
gnc_pop_locale (LC_NUMERIC, locale);
return retval;
return std::optional<double>{retval};
}
std::string
std::optional<std::string>
GncDbiSqlResult::IteratorImpl::get_string_at_col(const char* col) const
{
auto type = dbi_result_get_field_type (m_inst->m_dbi_result, col);
dbi_result_get_field_attribs (m_inst->m_dbi_result, col);
if(type != DBI_TYPE_STRING)
throw (std::invalid_argument{"Requested string from non-string column."});
return std::nullopt;
auto strval = dbi_result_get_string(m_inst->m_dbi_result, col);
if (strval == nullptr)
{
throw (std::invalid_argument{"Column empty."});
}
auto retval = std::string{strval};
return retval;
return std::optional<std::string>{strval ? strval : ""};
}
time64
std::optional<time64>
GncDbiSqlResult::IteratorImpl::get_time64_at_col (const char* col) const
{
auto result = (dbi_result_t*) (m_inst->m_dbi_result);
auto type = dbi_result_get_field_type (result, col);
dbi_result_get_field_attribs (result, col);
if (type != DBI_TYPE_DATETIME)
throw (std::invalid_argument{"Requested time64 from non-time64 column."});
return std::nullopt;
#if HAVE_LIBDBI_TO_LONGLONG
/* A less evil hack than the one required by libdbi-0.8, but
* still necessary to work around the same bug.
*/
auto retval = dbi_result_get_as_longlong(result, col);
auto timeval = dbi_result_get_as_longlong(result, col);
#else
/* A seriously evil hack to work around libdbi bug #15
* https://sourceforge.net/p/libdbi/bugs/15/. When libdbi
@ -174,11 +171,11 @@ GncDbiSqlResult::IteratorImpl::get_time64_at_col (const char* col) const
*/
auto row = dbi_result_get_currow (result);
auto idx = dbi_result_get_field_idx (result, col) - 1;
time64 retval = result->rows[row]->field_values[idx].d_datetime;
time64 timeval = result->rows[row]->field_values[idx].d_datetime;
#endif //HAVE_LIBDBI_TO_LONGLONG
if (retval < MINTIME || retval > MAXTIME)
retval = 0;
return retval;
if (timeval < MINTIME || timeval > MAXTIME)
timeval = 0;
return std::optional<time64>(timeval);
}

View File

@ -25,6 +25,8 @@
#ifndef __GNC_DBISQLBACKEND_HPP__
#define __GNC_DBISQLBACKEND_HPP__
#include <optional>
#include "gnc-backend-dbi.h"
#include <gnc-sql-result.hpp>
@ -53,11 +55,11 @@ protected:
virtual GncSqlRow& operator++();
virtual GncSqlRow& operator++(int) { return ++(*this); };
virtual GncSqlResult* operator*() { return m_inst; }
virtual int64_t get_int_at_col (const char* col) const;
virtual double get_float_at_col (const char* col) const;
virtual double get_double_at_col (const char* col) const;
virtual std::string get_string_at_col (const char* col)const;
virtual time64 get_time64_at_col (const char* col) const;
virtual std::optional<int64_t> get_int_at_col (const char* col) const;
virtual std::optional<double> get_float_at_col (const char* col) const;
virtual std::optional<double> get_double_at_col (const char* col) const;
virtual std::optional<std::string> get_string_at_col (const char* col)const;
virtual std::optional<time64> get_time64_at_col (const char* col) const;
virtual bool is_col_null(const char* col) const noexcept
{
return dbi_result_field_is_null(m_inst->m_dbi_result, col);

View File

@ -85,18 +85,13 @@ GncSqlColumnTableEntryImpl<CT_ADDRESS>::load (const GncSqlBackend* sql_be,
for (auto const& subtable_row : col_table)
{
auto buf = std::string{m_col_name} + "_" + subtable_row->m_col_name;
try
{
auto val = row.get_string_at_col (buf.c_str());
auto sub_setter = subtable_row->get_setter(GNC_ID_ADDRESS);
set_parameter (addr, val.c_str(), sub_setter,
auto val = row.get_string_at_col (buf.c_str());
auto sub_setter = subtable_row->get_setter(GNC_ID_ADDRESS);
if (val)
set_parameter (addr, val->c_str(), sub_setter,
subtable_row->m_gobj_param_name);
}
catch (std::invalid_argument&)
{
return;
}
}
set_parameter (pObject, addr,
reinterpret_cast<AddressSetterFunc>(get_setter(obj_name)),
m_gobj_param_name);

View File

@ -64,10 +64,10 @@ GncSqlColumnTableEntryImpl<CT_OWNERREF>::load (const GncSqlBackend* sql_be,
auto buf = std::string{m_col_name} + "_type";
try
{
type = static_cast<decltype(type)>(row.get_int_at_col (buf.c_str()));
type = static_cast<decltype(type)>(row.get_int_at_col(buf.c_str()).value_or(0));
buf = std::string{m_col_name} + "_guid";
auto val = row.get_string_at_col (buf.c_str());
if (string_to_guid (val.c_str(), &guid))
if (val && string_to_guid (val->c_str(), &guid))
pGuid = &guid;
}
catch (std::invalid_argument&)
@ -76,7 +76,7 @@ GncSqlColumnTableEntryImpl<CT_OWNERREF>::load (const GncSqlBackend* sql_be,
}
if (type == GNC_OWNER_NONE || pGuid == nullptr)
return;
switch (type)
{
case GNC_OWNER_CUSTOMER:

View File

@ -676,19 +676,12 @@ gnc_sql_slots_delete (GncSqlBackend* sql_be, const GncGUID* guid)
auto result = sql_be->execute_select_statement(stmt);
for (auto row : *result)
{
try
{
const GncSqlColumnTableEntryPtr table_row =
const GncSqlColumnTableEntryPtr table_row =
col_table[guid_val_col];
GncGUID child_guid;
auto val = row.get_string_at_col (table_row->name());
if (string_to_guid (val.c_str(), &child_guid))
gnc_sql_slots_delete (sql_be, &child_guid);
}
catch (std::invalid_argument&)
{
continue;
}
GncGUID child_guid;
auto val = row.get_string_at_col (table_row->name());
if (val && string_to_guid (val->c_str(), &child_guid))
gnc_sql_slots_delete (sql_be, &child_guid);
}
}

View File

@ -669,8 +669,9 @@ GncSqlBackend::init_version_info() noexcept
for (const auto& row : *result)
{
auto name = row.get_string_at_col (TABLE_COL_NAME);
unsigned int version = row.get_int_at_col (VERSION_COL_NAME);
m_versions.push_back(std::make_pair(name, version));
auto version = row.get_int_at_col (VERSION_COL_NAME);
if (name && version)
m_versions.push_back(std::make_pair(*name, static_cast<unsigned int>(*version)));
}
}
else

View File

@ -124,12 +124,9 @@ GncSqlColumnTableEntryImpl<CT_STRING>::load (const GncSqlBackend* sql_be,
g_return_if_fail (pObject != NULL);
g_return_if_fail (m_gobj_param_name != NULL || get_setter(obj_name) != NULL);
try
{
auto s = row.get_string_at_col (m_col_name);
set_parameter(pObject, s.c_str(), get_setter(obj_name), m_gobj_param_name);
}
catch (std::invalid_argument&) {}
auto s = row.get_string_at_col (m_col_name);
if (s)
set_parameter(pObject, s->c_str(), get_setter(obj_name), m_gobj_param_name);
}
template<> void
@ -174,8 +171,10 @@ GncSqlColumnTableEntryImpl<CT_INT>::load (const GncSqlBackend* sql_be,
g_return_if_fail (m_gobj_param_name != NULL || get_setter(obj_name) != NULL);
auto val = row.get_int_at_col(m_col_name);
set_parameter(pObject, val,
reinterpret_cast<IntSetterFunc>(get_setter(obj_name)), m_gobj_param_name);
if (val)
set_parameter(pObject, *val,
reinterpret_cast<IntSetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
}
template<> void
@ -208,9 +207,10 @@ GncSqlColumnTableEntryImpl<CT_BOOLEAN>::load (const GncSqlBackend* sql_be,
g_return_if_fail (m_gobj_param_name != NULL || get_setter(obj_name) != NULL);
auto val = row.get_int_at_col (m_col_name);
set_parameter(pObject, static_cast<int>(val),
reinterpret_cast<BooleanSetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
if (val)
set_parameter(pObject, static_cast<int>(*val),
reinterpret_cast<BooleanSetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
}
template<> void
@ -242,9 +242,10 @@ GncSqlColumnTableEntryImpl<CT_INT64>::load (const GncSqlBackend* sql_be,
g_return_if_fail (m_gobj_param_name != nullptr || get_setter(obj_name) != nullptr);
auto val = row.get_int_at_col (m_col_name);
set_parameter(pObject, val,
reinterpret_cast<Int64SetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
if (val)
set_parameter(pObject, *val,
reinterpret_cast<Int64SetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
}
template<> void
@ -273,29 +274,15 @@ GncSqlColumnTableEntryImpl<CT_DOUBLE>::load (const GncSqlBackend* sql_be,
{
g_return_if_fail (pObject != NULL);
g_return_if_fail (m_gobj_param_name != nullptr || get_setter(obj_name) != nullptr);
double val;
try
{
val = static_cast<double>(row.get_int_at_col(m_col_name));
}
catch (std::invalid_argument&)
{
try
{
val = row.get_float_at_col(m_col_name);
}
catch (std::invalid_argument&)
{
try
{
val = row.get_double_at_col(m_col_name);
}
catch (std::invalid_argument&)
{
val = 0.0;
}
}
}
double val{0.0};
if (auto int_val{row.get_int_at_col(m_col_name)})
val = static_cast<decltype(val)>(*int_val);
else if (auto float_val{row.get_float_at_col(m_col_name)})
val = static_cast<decltype(val)>(*float_val);
else if (auto double_val{row.get_double_at_col(m_col_name)})
val = *double_val;
set_parameter(pObject, val, get_setter(obj_name), m_gobj_param_name);
}
@ -329,16 +316,8 @@ GncSqlColumnTableEntryImpl<CT_GUID>::load (const GncSqlBackend* sql_be,
g_return_if_fail (pObject != NULL);
g_return_if_fail (m_gobj_param_name != nullptr || get_setter(obj_name) != nullptr);
std::string str;
try
{
str = row.get_string_at_col(m_col_name);
}
catch (std::invalid_argument&)
{
return;
}
if (string_to_guid (str.c_str(), &guid))
auto strval{row.get_string_at_col(m_col_name)};
if (strval && string_to_guid (strval->c_str(), &guid))
set_parameter(pObject, &guid, get_setter(obj_name), m_gobj_param_name);
}
@ -378,28 +357,28 @@ GncSqlColumnTableEntryImpl<CT_TIME>::load (const GncSqlBackend* sql_be,
{
time64 t{0};
g_return_if_fail (m_gobj_param_name != nullptr || get_setter(obj_name) != nullptr);
try
auto strval = row.get_string_at_col(m_col_name);
if (strval)
{
t = row.get_time64_at_col (m_col_name);
}
catch (std::invalid_argument&)
{
try
{
auto val = row.get_string_at_col(m_col_name);
GncDateTime time(val);
t = static_cast<time64>(time);
}
catch (const std::invalid_argument& err)
{
if (strcmp(err.what(), "Column empty.") != 0)
if (!strval->empty())
try
{
auto val = row.get_string_at_col (m_col_name);
PWARN("An invalid date %s was found in your database."
"It has been set to 1 January 1970.", val.c_str());
GncDateTime time(*strval);
t = static_cast<time64>(time);
}
catch (const std::invalid_argument& err)
{
PWARN("An invalid date %s was found in your database."
"It has been set to 1 January 1970.",
strval->c_str());
}
}
}
else
{
if (auto time64val = row.get_time64_at_col (m_col_name))
t = *time64val;
}
if (m_gobj_param_name != nullptr)
{
Time64 t64{t};
@ -472,37 +451,35 @@ GncSqlColumnTableEntryImpl<CT_GDATE>::load (const GncSqlBackend* sql_be,
return;
GDate date;
g_date_clear (&date, 1);
try
auto strval{row.get_string_at_col(m_col_name)};
if (strval)
{
if (strval->empty())
return;
auto year = static_cast<GDateYear>(stoi (strval->substr (0,4)));
auto month = static_cast<GDateMonth>(stoi (strval->substr (4,2)));
auto day = static_cast<GDateDay>(stoi (strval->substr (6,2)));
if (year != 0 || month != 0 || day != (GDateDay)0)
g_date_set_dmy(&date, day, month, year);
}
else
{
auto timeval = row.get_time64_at_col(m_col_name);
if (!timeval)
return;
/* time64_to_gdate applies the tz, and gdates are saved
* as ymd, so we don't want that.
*/
auto time = row.get_time64_at_col(m_col_name);
auto time = *timeval;
auto tm = gnc_gmtime(&time);
g_date_set_dmy(&date, tm->tm_mday,
static_cast<GDateMonth>(tm->tm_mon + 1),
tm->tm_year + 1900);
free(tm);
}
catch (std::invalid_argument&)
{
try
{
std::string str = row.get_string_at_col(m_col_name);
if (str.empty()) return;
auto year = static_cast<GDateYear>(stoi (str.substr (0,4)));
auto month = static_cast<GDateMonth>(stoi (str.substr (4,2)));
auto day = static_cast<GDateDay>(stoi (str.substr (6,2)));
if (year != 0 || month != 0 || day != (GDateDay)0)
g_date_set_dmy(&date, day, month, year);
}
catch (std::invalid_argument&)
{
return;
}
}
set_parameter(pObject, &date, get_setter(obj_name), m_gobj_param_name);
}
@ -562,24 +539,21 @@ GncSqlColumnTableEntryImpl<CT_NUMERIC>::load (const GncSqlBackend* sql_be,
g_return_if_fail (pObject != NULL);
g_return_if_fail (m_gobj_param_name != nullptr || get_setter(obj_name) != nullptr);
gnc_numeric n;
try
auto buf = g_strdup_printf ("%s_num", m_col_name);
auto num = row.get_int_at_col (buf);
g_free (buf);
buf = g_strdup_printf ("%s_denom", m_col_name);
auto denom = row.get_int_at_col (buf);
g_free (buf);
if (num && denom)
{
auto buf = g_strdup_printf ("%s_num", m_col_name);
auto num = row.get_int_at_col (buf);
g_free (buf);
buf = g_strdup_printf ("%s_denom", m_col_name);
auto denom = row.get_int_at_col (buf);
n = gnc_numeric_create (num, denom);
g_free (buf);
auto n = gnc_numeric_create (*num, *denom);
set_parameter(pObject, n,
reinterpret_cast<NumericSetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
}
catch (std::invalid_argument&)
{
return;
}
set_parameter(pObject, n,
reinterpret_cast<NumericSetterFunc>(get_setter(obj_name)),
m_gobj_param_name);
}
template<> void

View File

@ -183,35 +183,37 @@ public:
QofIdTypeConst obj_name,
void* pObject, T get_ref)
const noexcept
{
static QofLogModule log_module = G_LOG_DOMAIN;
g_return_if_fail (pObject != NULL);
{
static QofLogModule log_module = G_LOG_DOMAIN;
g_return_if_fail (pObject != NULL);
try
{
GncGUID guid;
auto val = row.get_string_at_col (m_col_name);
if (string_to_guid (val.c_str(), &guid))
{
auto target = get_ref(&guid);
if (target != nullptr)
set_parameter (pObject, target, get_setter(obj_name),
m_gobj_param_name);
else
DEBUG("GUID %s returned null %s reference.",
val.c_str(), m_gobj_param_name);
}
else
{
if (val.empty()) DEBUG("Can't load empty guid string for column %s", m_col_name);
else DEBUG("Invalid GUID %s for column %s", val.c_str(), m_col_name);
}
}
catch (std::invalid_argument& err) {
DEBUG("set_parameter threw %s for column %s", err.what(), m_col_name);
}
GncGUID guid;
auto val = row.get_string_at_col (m_col_name);
if (!val)
{
DEBUG("set parameter: No string in column %s.", m_col_name);
return;
}
if (string_to_guid (val->c_str(), &guid))
{
auto target = get_ref(&guid);
if (target != nullptr)
set_parameter (pObject, target, get_setter(obj_name),
m_gobj_param_name);
else
DEBUG("GUID %s returned null %s reference.",
val->c_str(), m_gobj_param_name);
}
else
{
if (val->empty())
DEBUG("Can't load empty guid string for column %s", m_col_name);
else
DEBUG("Invalid GUID %s for column %s", val->c_str(), m_col_name);
}
}
protected:
template <typename T> T

View File

@ -27,6 +27,7 @@
#include <qof.h>
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
@ -49,11 +50,11 @@ protected:
virtual ~IteratorImpl() = default;
virtual GncSqlRow& operator++() = 0;
virtual GncSqlResult* operator*() = 0;
virtual int64_t get_int_at_col (const char* col) const = 0;
virtual double get_float_at_col (const char* col) const = 0;
virtual double get_double_at_col (const char* col) const = 0;
virtual std::string get_string_at_col (const char* col) const = 0;
virtual time64 get_time64_at_col (const char* col) const = 0;
virtual std::optional<int64_t> get_int_at_col (const char* col) const = 0;
virtual std::optional<double> get_float_at_col (const char* col) const = 0;
virtual std::optional<double> get_double_at_col (const char* col) const = 0;
virtual std::optional<std::string> get_string_at_col (const char* col) const = 0;
virtual std::optional<time64> get_time64_at_col (const char* col) const = 0;
virtual bool is_col_null (const char* col) const noexcept = 0;
};
};
@ -84,15 +85,15 @@ public:
GncSqlRow& operator++();
GncSqlRow& operator*() { return *this; }
friend bool operator!=(const GncSqlRow&, const GncSqlRow&);
int64_t get_int_at_col (const char* col) const {
std::optional<int64_t> get_int_at_col (const char* col) const {
return m_iter->get_int_at_col (col); }
double get_float_at_col (const char* col) const {
std::optional<double> get_float_at_col (const char* col) const {
return m_iter->get_float_at_col (col); }
double get_double_at_col (const char* col) const {
std::optional<double> get_double_at_col (const char* col) const {
return m_iter->get_double_at_col (col); }
std::string get_string_at_col (const char* col) const {
std::optional<std::string> get_string_at_col (const char* col) const {
return m_iter->get_string_at_col (col); }
time64 get_time64_at_col (const char* col) const {
std::optional<time64> get_time64_at_col (const char* col) const {
return m_iter->get_time64_at_col (col); }
bool is_col_null (const char* col) const noexcept {
return m_iter->is_col_null (col); }

View File

@ -792,27 +792,27 @@ GncSqlColumnTableEntryImpl<CT_TXREF>::load (const GncSqlBackend* sql_be,
g_return_if_fail (sql_be != NULL);
g_return_if_fail (pObject != NULL);
try
auto val = row.get_string_at_col (m_col_name);
if (!val)
return;
GncGUID guid;
Transaction *tx = nullptr;
if (string_to_guid (val->c_str(), &guid))
tx = xaccTransLookup (&guid, sql_be->book());
// If the transaction is not found, try loading it
std::string tpkey(tx_col_table[0]->name());
if (tx == nullptr)
{
auto val = row.get_string_at_col (m_col_name);
GncGUID guid;
Transaction *tx = nullptr;
if (string_to_guid (val.c_str(), &guid))
tx = xaccTransLookup (&guid, sql_be->book());
// If the transaction is not found, try loading it
std::string tpkey(tx_col_table[0]->name());
if (tx == nullptr)
{
std::string sql = tpkey + " = '" + val + "'";
query_transactions ((GncSqlBackend*)sql_be, sql);
tx = xaccTransLookup (&guid, sql_be->book());
}
if (tx != nullptr)
set_parameter (pObject, tx, get_setter(obj_name), m_gobj_param_name);
std::string sql = tpkey + " = '" + *val + "'";
query_transactions ((GncSqlBackend*)sql_be, sql);
tx = xaccTransLookup (&guid, sql_be->book());
}
catch (std::invalid_argument&) {}
if (tx != nullptr)
set_parameter (pObject, tx, get_setter(obj_name), m_gobj_param_name);
}
template<> void

View File

@ -62,15 +62,15 @@ protected:
virtual GncSqlRow& operator++() { return m_inst->m_row; }
virtual GncSqlRow& operator++(int) { return ++(*this); };
virtual GncSqlResult* operator*() { return m_inst; }
virtual int64_t get_int_at_col (const char* col) const
virtual std::optional<int64_t> get_int_at_col (const char* col) const
{ return 1LL; }
virtual double get_float_at_col (const char* col) const
virtual std::optional<double> get_float_at_col (const char* col) const
{ return 1.0; }
virtual double get_double_at_col (const char* col) const
virtual std::optional<double> get_double_at_col (const char* col) const
{ return 1.0; }
virtual std::string get_string_at_col (const char* col)const
virtual std::optional<std::string> get_string_at_col (const char* col)const
{ return std::string{"foo"}; }
virtual time64 get_time64_at_col (const char* col) const
virtual std::optional<time64> get_time64_at_col (const char* col) const
{ return 1466270857LL; }
virtual bool is_col_null(const char* col) const noexcept
{ return false; }