IntenseFinance/gnucash
3
0
Fork 0
mirror of https://github.com/Gnucash/gnucash.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Bug 772776 - VERY large queries (over 11000 fields in IN clause) slow...

Browse Source 
down GnuCash

Replace with joins or subqueries. Affords a 20% speedup on Windows.
This commit is contained in:
John Ralls
2018-03-24 16:30:34 -07:00
parent e90a662a20
commit 01420adb99
5 changed files with 62 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
libgnucash/backend/sql/gnc-slots-sql.cpp
View File

@@ -821,44 +821,6 @@ load_slot_for_list_item (GncSqlBackend* sql_be, GncSqlRow& row,
}
void
gnc_sql_slots_load_for_instancevec (GncSqlBackend* sql_be, InstanceVec& instances)
{
QofCollection* coll;
std::stringstream sql;
g_return_if_fail (sql_be != NULL);
// Ignore empty list
if (instances.empty()) return;
coll = qof_instance_get_collection (instances[0]);
// Create the query for all slots for all items on the list
sql << "SELECT * FROM " << TABLE_NAME << " WHERE " <<
obj_guid_col_table[0]->name();
if (instances.size() != 1)
sql << " IN (";
else
sql << " = ";
gnc_sql_append_guids_to_sql (sql, instances);
if (instances.size() > 1)
sql << ")";
// Execute the query and load the slots
auto stmt = sql_be->create_statement_from_sql(sql.str());
if (stmt == nullptr)
{
PERR ("stmt == NULL, SQL = '%s'\n", sql.str().c_str());
return;
}
auto result = sql_be->execute_select_statement (stmt);
for (auto row : *result)
load_slot_for_list_item (sql_be, row, coll);
}
static void
load_slot_for_book_object (GncSqlBackend* sql_be, GncSqlRow& row,
BookLookupFn lookup_fn)

11
libgnucash/backend/sql/gnc-slots-sql.h
View File

@@ -76,17 +76,6 @@ gboolean gnc_sql_slots_delete (GncSqlBackend* sql_be, const GncGUID* guid);
*/
void gnc_sql_slots_load (GncSqlBackend* sql_be, QofInstance* inst);
/**
* gnc_sql_slots_load_for_instancevec - Loads slots for a set of QofInstance*
* from the db. Loading slots for a set is faster than loading for one object
* at a time because fewer SQL queries are used.
*
* @param sql_be SQL backend
* @param list List of objects
*/
void gnc_sql_slots_load_for_instancevec (GncSqlBackend* sql_be,
InstanceVec& instances);
typedef QofInstance* (*BookLookupFn) (const GncGUID* guid,
const QofBook* book);

12
libgnucash/backend/sql/gnc-sql-backend.cpp
View File

@@ -510,19 +510,19 @@ GncSqlBackend::sync(QofBook* book)
void
GncSqlBackend::begin(QofInstance* inst)
{
g_return_if_fail (inst != NULL);
//g_return_if_fail (inst != NULL);
ENTER (" ");
LEAVE ("");
//ENTER (" ");
//LEAVE ("");
}
void
GncSqlBackend::rollback(QofInstance* inst)
{
g_return_if_fail (inst != NULL);
//g_return_if_fail (inst != NULL);
ENTER (" ");
LEAVE ("");
//ENTER (" ");
//LEAVE ("");
}
void

147
libgnucash/backend/sql/gnc-transaction-sql.cpp
View File

@@ -247,52 +247,36 @@ load_single_split (GncSqlBackend* sql_be, GncSqlRow& row)
}
return pSplit;
}
static void
load_splits_for_sql_subquery (GncSqlBackend* sql_be, std::string subquery)
load_splits_for_transactions (GncSqlBackend* sql_be, std::string selector)
{
g_return_if_fail (sql_be != NULL);
const std::string sskey(tx_guid_col_table[0]->name());
std::string sql("SELECT * FROM " SPLIT_TABLE " WHERE ");
sql += sskey + " IN (" + subquery + ")";
auto stmt = sql_be->create_statement_from_sql(sql);
auto result = sql_be->execute_select_statement(stmt);
for (auto row : *result)
load_single_split (sql_be, row);
const std::string spkey(split_col_table[0]->name());
std::string sub_subquery("SELECT DISTINCT ");
sub_subquery += spkey + " FROM " SPLIT_TABLE " WHERE " + sskey +
" IN (" + subquery + ")";
gnc_sql_slots_load_for_sql_subquery(sql_be, sub_subquery,
(BookLookupFn)xaccSplitLookup);
}
const std::string sskey(tx_guid_col_table[0]->name());
const std::string tpkey(tx_col_table[0]->name());
static void
load_splits_for_tx_list (GncSqlBackend* sql_be, InstanceVec& transactions)
{
g_return_if_fail (sql_be != NULL);
std::stringstream sql;
sql << "SELECT * FROM " << SPLIT_TABLE << " WHERE " <<
tx_guid_col_table[0]->name() << " IN (";
gnc_sql_append_guids_to_sql (sql, transactions);
sql << ")";
std::string sql("SELECT ");
if (selector.empty())
{
sql += SPLIT_TABLE ".* FROM " SPLIT_TABLE " INNER JOIN "
TRANSACTION_TABLE " WHERE " SPLIT_TABLE "." + sskey + " = "
TRANSACTION_TABLE "." + tpkey;
selector = "(SELECT DISTINCT " + tpkey + " FROM " TRANSACTION_TABLE ")";
}
else
sql += " * FROM " SPLIT_TABLE " WHERE " + sskey + " IN " + selector;
// Execute the query and load the splits
auto stmt = sql_be->create_statement_from_sql(sql.str());
auto stmt = sql_be->create_statement_from_sql(sql);
auto result = sql_be->execute_select_statement (stmt);
InstanceVec instances;
for (auto row : *result)
{
Split* s = load_single_split (sql_be, row);
if (s != nullptr)
instances.push_back(QOF_INSTANCE(s));
}
if (!instances.empty())
gnc_sql_slots_load_for_instancevec (sql_be, instances);
sql = "SELECT DISTINCT ";
sql += spkey + " FROM " SPLIT_TABLE " WHERE " + sskey + " IN " + selector;
gnc_sql_slots_load_for_sql_subquery(sql_be, sql,
(BookLookupFn)xaccSplitLookup);
}
static Transaction*
@@ -356,19 +340,30 @@ typedef struct
* @param stmt SQL statement
*/
static void
query_transactions (GncSqlBackend* sql_be, const GncSqlStatementPtr& stmt)
query_transactions (GncSqlBackend* sql_be, std::string selector)
{
g_return_if_fail (sql_be != NULL);
g_return_if_fail (stmt != NULL);
const std::string tpkey(tx_col_table[0]->name());
std::string sql("SELECT * FROM " TRANSACTION_TABLE);
if (!selector.empty() && selector[0] == '(')
sql += " WHERE " + tpkey + " IN " + selector;
else if (!selector.empty()) // plain condition
sql += " WHERE " + selector;
auto stmt = sql_be->create_statement_from_sql(sql);
auto result = sql_be->execute_select_statement(stmt);
if (result->begin() == result->end())
{
PINFO("Query %s returned no results", sql.c_str());
return;
}
Transaction* tx;
// Load the transactions
InstanceVec instances;
instances.reserve(result->size());
for (auto row : *result)
{
tx = load_single_tx (sql_be, row);
@@ -382,8 +377,15 @@ query_transactions (GncSqlBackend* sql_be, const GncSqlStatementPtr& stmt)
// Load all splits and slots for the transactions
if (!instances.empty())
{
gnc_sql_slots_load_for_instancevec (sql_be, instances);
load_splits_for_tx_list (sql_be, instances);
const std::string tpkey(tx_col_table[0]->name());
if (selector.empty())
{
selector = "(SELECT DISTINCT ";
selector += tpkey + " FROM " TRANSACTION_TABLE +")";
}
load_splits_for_transactions (sql_be, selector);
gnc_sql_slots_load_for_sql_subquery (sql_be, selector,
(BookLookupFn)xaccTransLookup);
}
// Commit all of the transactions
@@ -691,16 +693,15 @@ void gnc_sql_transaction_load_tx_for_account (GncSqlBackend* sql_be,
g_return_if_fail (account != NULL);
guid = qof_instance_get_guid (QOF_INSTANCE (account));
(void)guid_to_string_buff (guid, guid_buf);
query_sql = g_strdup_printf (
"SELECT DISTINCT t.* FROM %s AS t, %s AS s WHERE s.tx_guid=t.guid AND s.account_guid ='%s'",
TRANSACTION_TABLE, SPLIT_TABLE, guid_buf);
auto stmt = sql_be->create_statement_from_sql(query_sql);
g_free (query_sql);
if (stmt != nullptr)
{
query_transactions (sql_be, stmt);
}
const std::string tpkey(tx_col_table[0]->name()); //guid
const std::string spkey(split_col_table[0]->name()); //guid
const std::string stkey(split_col_table[1]->name()); //txn_guid
const std::string sakey(split_col_table[2]->name()); //account_guid
std::string sql("(SELECT DISTINCT ");
sql += stkey + " FROM " SPLIT_TABLE " WHERE " + sakey + " = '";
sql += gnc::GUID(*guid).to_string() + "')";
query_transactions (sql_be, sql);
}
/**
@@ -713,44 +714,7 @@ void
GncSqlTransBackend::load_all (GncSqlBackend* sql_be)
{
g_return_if_fail (sql_be != NULL);
std::string query_sql("SELECT * FROM " TRANSACTION_TABLE);
auto stmt = sql_be->create_statement_from_sql(query_sql);
if (stmt != nullptr)
{
auto result = sql_be->execute_select_statement(stmt);
if (result->begin() == result->end())
return;
Transaction* tx;
// Load the transactions
InstanceVec instances;
instances.reserve(result->size());
for (auto row : *result)
{
tx = load_single_tx (sql_be, row);
if (tx != nullptr)
{
xaccTransScrubPostedDate (tx);
instances.push_back(QOF_INSTANCE(tx));
}
}
if (instances.empty())
return;
const std::string tpkey(tx_col_table[0]->name());
std::string subquery("SELECT DISTINCT ");
subquery += tpkey + " FROM " TRANSACTION_TABLE;
gnc_sql_slots_load_for_sql_subquery (sql_be, subquery,
(BookLookupFn)xaccTransLookup);
load_splits_for_sql_subquery (sql_be, subquery);
// Commit all of the transactions
for (auto instance : instances)
xaccTransCommitEdit(GNC_TRANSACTION(instance));
}
query_transactions (sql_be, "");
}
static void
@@ -995,12 +959,11 @@ GncSqlColumnTableEntryImpl<CT_TXREF>::load (const GncSqlBackend* sql_be,
tx = xaccTransLookup (&guid, sql_be->book());
// If the transaction is not found, try loading it
std::string tpkey(tx_col_table[0]->name());
if (tx == nullptr)
{
auto buf = std::string{"SELECT * FROM "} + TRANSACTION_TABLE +
" WHERE guid='" + val + "'";
auto stmt = sql_be->create_statement_from_sql (buf);
query_transactions ((GncSqlBackend*)sql_be, stmt);
std::string sql = tpkey + " = '" + val + "'";
query_transactions ((GncSqlBackend*)sql_be, sql);
tx = xaccTransLookup (&guid, sql_be->book());
}

2
libgnucash/engine/TransLog.c
View File

@@ -230,7 +230,7 @@ xaccTransWriteLog (Transaction *trans, char flag)
if (!gen_logs)
{
PINFO ("Attempt to write disabled transaction log");
PINFO ("Attempt to write disabled transaction log");
return;
}
if (!trans_log) return;