IntenseFinance/gnucash
3
0
Fork 0
mirror of https://github.com/Gnucash/gnucash.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Bug 772776 - VERY large queries (over 11000 fields in IN clause) slow...

Browse Source 
down GnuCash

Replace with joins or subqueries. Affords a 20% speedup on Windows.
This commit is contained in:
John Ralls
2018-03-24 16:30:34 -07:00
parent e90a662a20
commit 01420adb99
5 changed files with 62 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
libgnucash/backend/sql/gnc-slots-sql.cpp
View File

@@ -821,44 +821,6 @@ load_slot_for_list_item (GncSqlBackend* sql_be, GncSqlRow& row,
} }
void
gnc_sql_slots_load_for_instancevec (GncSqlBackend* sql_be, InstanceVec& instances)
{
QofCollection* coll;
std::stringstream sql;
g_return_if_fail (sql_be != NULL);
// Ignore empty list
if (instances.empty()) return;
coll = qof_instance_get_collection (instances[0]);
// Create the query for all slots for all items on the list
sql << "SELECT * FROM " << TABLE_NAME << " WHERE " <<
obj_guid_col_table[0]->name();
if (instances.size() != 1)
sql << " IN (";
else
sql << " = ";
gnc_sql_append_guids_to_sql (sql, instances);
if (instances.size() > 1)
sql << ")";
// Execute the query and load the slots
auto stmt = sql_be->create_statement_from_sql(sql.str());
if (stmt == nullptr)
{
PERR ("stmt == NULL, SQL = '%s'\n", sql.str().c_str());
return;
}
auto result = sql_be->execute_select_statement (stmt);
for (auto row : *result)
load_slot_for_list_item (sql_be, row, coll);
}
static void static void
load_slot_for_book_object (GncSqlBackend* sql_be, GncSqlRow& row, load_slot_for_book_object (GncSqlBackend* sql_be, GncSqlRow& row,
BookLookupFn lookup_fn) BookLookupFn lookup_fn)

11
libgnucash/backend/sql/gnc-slots-sql.h
View File

@@ -76,17 +76,6 @@ gboolean gnc_sql_slots_delete (GncSqlBackend* sql_be, const GncGUID* guid);
*/ */
void gnc_sql_slots_load (GncSqlBackend* sql_be, QofInstance* inst); void gnc_sql_slots_load (GncSqlBackend* sql_be, QofInstance* inst);
/**
* gnc_sql_slots_load_for_instancevec - Loads slots for a set of QofInstance*
* from the db. Loading slots for a set is faster than loading for one object
* at a time because fewer SQL queries are used.
*
* @param sql_be SQL backend
* @param list List of objects
*/
void gnc_sql_slots_load_for_instancevec (GncSqlBackend* sql_be,
InstanceVec& instances);
typedef QofInstance* (*BookLookupFn) (const GncGUID* guid, typedef QofInstance* (*BookLookupFn) (const GncGUID* guid,
const QofBook* book); const QofBook* book);

12
libgnucash/backend/sql/gnc-sql-backend.cpp
View File

@@ -510,19 +510,19 @@ GncSqlBackend::sync(QofBook* book)
void void
GncSqlBackend::begin(QofInstance* inst) GncSqlBackend::begin(QofInstance* inst)
{ {
g_return_if_fail (inst != NULL); //g_return_if_fail (inst != NULL);
ENTER (" "); //ENTER (" ");
LEAVE (""); //LEAVE ("");
} }
void void
GncSqlBackend::rollback(QofInstance* inst) GncSqlBackend::rollback(QofInstance* inst)
{ {
g_return_if_fail (inst != NULL); //g_return_if_fail (inst != NULL);
ENTER (" "); //ENTER (" ");
LEAVE (""); //LEAVE ("");
} }
void void

147
libgnucash/backend/sql/gnc-transaction-sql.cpp
View File

@@ -247,52 +247,36 @@ load_single_split (GncSqlBackend* sql_be, GncSqlRow& row)
} }
return pSplit; return pSplit;
} }
static void static void
load_splits_for_sql_subquery (GncSqlBackend* sql_be, std::string subquery) load_splits_for_transactions (GncSqlBackend* sql_be, std::string selector)
{ {
g_return_if_fail (sql_be != NULL); g_return_if_fail (sql_be != NULL);
const std::string sskey(tx_guid_col_table[0]->name());
std::string sql("SELECT * FROM " SPLIT_TABLE " WHERE ");
sql += sskey + " IN (" + subquery + ")";
auto stmt = sql_be->create_statement_from_sql(sql);
auto result = sql_be->execute_select_statement(stmt);
for (auto row : *result)
load_single_split (sql_be, row);
const std::string spkey(split_col_table[0]->name()); const std::string spkey(split_col_table[0]->name());
std::string sub_subquery("SELECT DISTINCT "); const std::string sskey(tx_guid_col_table[0]->name());
sub_subquery += spkey + " FROM " SPLIT_TABLE " WHERE " + sskey + const std::string tpkey(tx_col_table[0]->name());
" IN (" + subquery + ")";
gnc_sql_slots_load_for_sql_subquery(sql_be, sub_subquery,
(BookLookupFn)xaccSplitLookup);
}
static void std::string sql("SELECT ");
load_splits_for_tx_list (GncSqlBackend* sql_be, InstanceVec& transactions) if (selector.empty())
{ {
g_return_if_fail (sql_be != NULL); sql += SPLIT_TABLE ".* FROM " SPLIT_TABLE " INNER JOIN "
TRANSACTION_TABLE " WHERE " SPLIT_TABLE "." + sskey + " = "
std::stringstream sql; TRANSACTION_TABLE "." + tpkey;
selector = "(SELECT DISTINCT " + tpkey + " FROM " TRANSACTION_TABLE ")";
sql << "SELECT * FROM " << SPLIT_TABLE << " WHERE " << }
tx_guid_col_table[0]->name() << " IN ("; else
gnc_sql_append_guids_to_sql (sql, transactions); sql += " * FROM " SPLIT_TABLE " WHERE " + sskey + " IN " + selector;
sql << ")";
// Execute the query and load the splits // Execute the query and load the splits
auto stmt = sql_be->create_statement_from_sql(sql.str()); auto stmt = sql_be->create_statement_from_sql(sql);
auto result = sql_be->execute_select_statement (stmt); auto result = sql_be->execute_select_statement (stmt);
InstanceVec instances;
for (auto row : *result) for (auto row : *result)
{
Split* s = load_single_split (sql_be, row); Split* s = load_single_split (sql_be, row);
if (s != nullptr) sql = "SELECT DISTINCT ";
instances.push_back(QOF_INSTANCE(s)); sql += spkey + " FROM " SPLIT_TABLE " WHERE " + sskey + " IN " + selector;
} gnc_sql_slots_load_for_sql_subquery(sql_be, sql,
(BookLookupFn)xaccSplitLookup);
if (!instances.empty())
gnc_sql_slots_load_for_instancevec (sql_be, instances);
} }
static Transaction* static Transaction*
@@ -356,19 +340,30 @@ typedef struct
* @param stmt SQL statement * @param stmt SQL statement
*/ */
static void static void
query_transactions (GncSqlBackend* sql_be, const GncSqlStatementPtr& stmt) query_transactions (GncSqlBackend* sql_be, std::string selector)
{ {
g_return_if_fail (sql_be != NULL); g_return_if_fail (sql_be != NULL);
g_return_if_fail (stmt != NULL);
const std::string tpkey(tx_col_table[0]->name());
std::string sql("SELECT * FROM " TRANSACTION_TABLE);
if (!selector.empty() && selector[0] == '(')
sql += " WHERE " + tpkey + " IN " + selector;
else if (!selector.empty()) // plain condition
sql += " WHERE " + selector;
auto stmt = sql_be->create_statement_from_sql(sql);
auto result = sql_be->execute_select_statement(stmt); auto result = sql_be->execute_select_statement(stmt);
if (result->begin() == result->end()) if (result->begin() == result->end())
{
PINFO("Query %s returned no results", sql.c_str());
return; return;
}
Transaction* tx; Transaction* tx;
// Load the transactions // Load the transactions
InstanceVec instances; InstanceVec instances;
instances.reserve(result->size());
for (auto row : *result) for (auto row : *result)
{ {
tx = load_single_tx (sql_be, row); tx = load_single_tx (sql_be, row);
@@ -382,8 +377,15 @@ query_transactions (GncSqlBackend* sql_be, const GncSqlStatementPtr& stmt)
// Load all splits and slots for the transactions // Load all splits and slots for the transactions
if (!instances.empty()) if (!instances.empty())
{ {
gnc_sql_slots_load_for_instancevec (sql_be, instances); const std::string tpkey(tx_col_table[0]->name());
load_splits_for_tx_list (sql_be, instances); if (selector.empty())
{
selector = "(SELECT DISTINCT ";
selector += tpkey + " FROM " TRANSACTION_TABLE +")";
}
load_splits_for_transactions (sql_be, selector);
gnc_sql_slots_load_for_sql_subquery (sql_be, selector,
(BookLookupFn)xaccTransLookup);
} }
// Commit all of the transactions // Commit all of the transactions
@@ -691,16 +693,15 @@ void gnc_sql_transaction_load_tx_for_account (GncSqlBackend* sql_be,
g_return_if_fail (account != NULL); g_return_if_fail (account != NULL);
guid = qof_instance_get_guid (QOF_INSTANCE (account)); guid = qof_instance_get_guid (QOF_INSTANCE (account));
(void)guid_to_string_buff (guid, guid_buf);
query_sql = g_strdup_printf ( const std::string tpkey(tx_col_table[0]->name()); //guid
"SELECT DISTINCT t.* FROM %s AS t, %s AS s WHERE s.tx_guid=t.guid AND s.account_guid ='%s'", const std::string spkey(split_col_table[0]->name()); //guid
TRANSACTION_TABLE, SPLIT_TABLE, guid_buf); const std::string stkey(split_col_table[1]->name()); //txn_guid
auto stmt = sql_be->create_statement_from_sql(query_sql); const std::string sakey(split_col_table[2]->name()); //account_guid
g_free (query_sql); std::string sql("(SELECT DISTINCT ");
if (stmt != nullptr) sql += stkey + " FROM " SPLIT_TABLE " WHERE " + sakey + " = '";
{ sql += gnc::GUID(*guid).to_string() + "')";
query_transactions (sql_be, stmt); query_transactions (sql_be, sql);
}
} }
/** /**
@@ -713,44 +714,7 @@ void
GncSqlTransBackend::load_all (GncSqlBackend* sql_be) GncSqlTransBackend::load_all (GncSqlBackend* sql_be)
{ {
g_return_if_fail (sql_be != NULL); g_return_if_fail (sql_be != NULL);
query_transactions (sql_be, "");
std::string query_sql("SELECT * FROM " TRANSACTION_TABLE);
auto stmt = sql_be->create_statement_from_sql(query_sql);
if (stmt != nullptr)
{
auto result = sql_be->execute_select_statement(stmt);
if (result->begin() == result->end())
return;
Transaction* tx;
// Load the transactions
InstanceVec instances;
instances.reserve(result->size());
for (auto row : *result)
{
tx = load_single_tx (sql_be, row);
if (tx != nullptr)
{
xaccTransScrubPostedDate (tx);
instances.push_back(QOF_INSTANCE(tx));
}
}
if (instances.empty())
return;
const std::string tpkey(tx_col_table[0]->name());
std::string subquery("SELECT DISTINCT ");
subquery += tpkey + " FROM " TRANSACTION_TABLE;
gnc_sql_slots_load_for_sql_subquery (sql_be, subquery,
(BookLookupFn)xaccTransLookup);
load_splits_for_sql_subquery (sql_be, subquery);
// Commit all of the transactions
for (auto instance : instances)
xaccTransCommitEdit(GNC_TRANSACTION(instance));
}
} }
static void static void
@@ -995,12 +959,11 @@ GncSqlColumnTableEntryImpl<CT_TXREF>::load (const GncSqlBackend* sql_be,
tx = xaccTransLookup (&guid, sql_be->book()); tx = xaccTransLookup (&guid, sql_be->book());
// If the transaction is not found, try loading it // If the transaction is not found, try loading it
std::string tpkey(tx_col_table[0]->name());
if (tx == nullptr) if (tx == nullptr)
{ {
auto buf = std::string{"SELECT * FROM "} + TRANSACTION_TABLE + std::string sql = tpkey + " = '" + val + "'";
" WHERE guid='" + val + "'"; query_transactions ((GncSqlBackend*)sql_be, sql);
auto stmt = sql_be->create_statement_from_sql (buf);
query_transactions ((GncSqlBackend*)sql_be, stmt);
tx = xaccTransLookup (&guid, sql_be->book()); tx = xaccTransLookup (&guid, sql_be->book());
} }

2
libgnucash/engine/TransLog.c
View File

@@ -230,7 +230,7 @@ xaccTransWriteLog (Transaction *trans, char flag)
if (!gen_logs) if (!gen_logs)
{ {
PINFO ("Attempt to write disabled transaction log"); PINFO ("Attempt to write disabled transaction log");
return; return;
} }
if (!trans_log) return; if (!trans_log) return;