performance improvements for kvp handling; should speed loads of

transactions, splits, considerably


git-svn-id: svn+ssh://svn.gnucash.org/repo/gnucash/trunk@5057 57a11ea4-9604-0410-9ed3-97b8803252fd
This commit is contained in:
Linas Vepstas
2001-07-29 05:54:35 +00:00
parent e43a518e51
commit a1e4ca2cfe
10 changed files with 140 additions and 122 deletions
+7
View File
@@ -247,6 +247,13 @@ the trick ... some different algorithm is needed.
Handy for raw-file-loading performance measurement is the the
script 'scan-acct.pl' in src/optional/swig/examples'
Weird shit: loading the same data, but in slightly different
order, can make a *huge* difference in the speed of balance
subtotal calculations (used only in multi-user mode.) Merely
rearranging the order of the splits in a transaction can cause
a factor of 20 (twenty) difference in performance. This can turn
a 4.5 second load into a minute & a half load !!!! Yow!
Investigating ...
To Be Done
----------
+17 -4
View File
@@ -109,6 +109,12 @@ pgendStoreAccountNoLock (PGBackend *be, Account *acct,
acct->version ++; /* be sure to update the version !! */
acct->version_check = be->version_check;
if ((0 == acct->idata) &&
(FALSE == kvp_frame_is_empty (xaccAccountGetSlots(acct))))
{
acct->idata = pgendNewGUIDidx(be);
}
pgendPutOneAccountOnly (be, acct);
/* make sure the account's commodity is in the commodity table */
@@ -119,7 +125,10 @@ pgendStoreAccountNoLock (PGBackend *be, Account *acct,
com = xaccAccountGetCommodity (acct);
pgendPutOneCommodityOnly (be, (gnc_commodity *) com);
pgendKVPStore (be, &(acct->guid), acct->kvp_data);
if (acct->idata)
{
pgendKVPStore (be, acct->idata, acct->kvp_data);
}
LEAVE(" ");
}
@@ -205,7 +214,8 @@ static gpointer
restore_cb (Account *acc, void * cb_data)
{
PGBackend *be = (PGBackend *) cb_data;
acc->kvp_data = pgendKVPFetch (be, &(acc->guid), acc->kvp_data);
if (0 == acc->idata) return NULL;
acc->kvp_data = pgendKVPFetch (be, acc->idata, acc->kvp_data);
return NULL;
}
@@ -377,7 +387,10 @@ pgendCopyAccountToEngine (PGBackend *be, const GUID *acct_guid)
acc = xaccAccountLookup (acct_guid);
/* restore any kvp data associated with the transaction and splits */
acc->kvp_data = pgendKVPFetch (be, &(acc->guid), acc->kvp_data);
if (acc->idata)
{
acc->kvp_data = pgendKVPFetch (be, acc->idata, acc->kvp_data);
}
acc->version_check = be->version_check;
}
@@ -449,7 +462,7 @@ pgend_account_commit_edit (Backend * bend,
{
const GUID *guid = xaccAccountGetGUID(acct);
pgendStoreAuditAccount (be, acct, SQL_DELETE);
pgendKVPDelete (be, guid);
pgendKVPDelete (be, acct->idata);
p = be->buff; *p = 0;
p = stpcpy (p, "DELETE FROM gncAccount WHERE accountGuid='");
+36 -85
View File
@@ -42,6 +42,28 @@
static short module = MOD_KVP;
/* =========================================================== */
/* get a unique iguid index */
static gpointer
iguid_cb (PGBackend *be, PGresult *result, int j, gpointer data)
{
int iguid = atoi (DB_GET_VAL ("iguid", 0));
return (gpointer) iguid;
}
guint32
pgendNewGUIDidx (PGBackend *be)
{
guint32 iguid;
char * p;
p = "SELECT nextval('gnc_iguid_seq') AS iguid;";
iguid = (guint32) pgendGetResults (be, iguid_cb, (gpointer) 0);
return iguid;
}
/* =========================================================== */
/* given integer ipath (path id) and a string, poke the string
* into a cache in local memory
@@ -91,17 +113,13 @@ pgendPeekPathCache (PGBackend *be, int ipath)
static gpointer
ival_cb (PGBackend *be, PGresult *result, int j, gpointer data)
{
int ival = atoi (DB_GET_VAL ((char *)data, 0));
int ival = atoi (DB_GET_VAL ("ipath", 0));
return (gpointer) ival;
}
static int
pgendGetCache (PGBackend *be,
const char *table_name,
const char *key_name,
const char *cache_name,
const char *val_str)
pgendGetCache (PGBackend *be, const char *val_str)
{
char *p;
int ival =0;
@@ -111,29 +129,19 @@ pgendGetCache (PGBackend *be,
/* first, lets see if we can find the guid or path.
* If we can then just return it */
p = be->buff; *p = 0;
p = stpcpy (p, "SELECT ");
p = stpcpy (p, cache_name);
p = stpcpy (p, " FROM ");
p = stpcpy (p, table_name);
p = stpcpy (p, " WHERE ");
p = stpcpy (p, key_name);
p = stpcpy (p, " ='");
p = stpcpy (p, "SELECT ipath FROM gncPathCache WHERE path='");
p = stpcpy (p, val_str);
p = stpcpy (p, "';");
SEND_QUERY (be,be->buff, 0);
ival = (int) pgendGetResults (be, ival_cb, (gpointer) cache_name);
if (ival && (ival != (int)cache_name)) return ival;
ival = (int) pgendGetResults (be, ival_cb, (gpointer) 0);
if (ival) return ival;
/* Else, this guid has never been stored before.
* Poke it into the the database */
p = be->buff; *p = 0;
p = stpcpy (p, "INSERT INTO ");
p = stpcpy (p, table_name);
p = stpcpy (p, " (");
p = stpcpy (p, key_name);
p = stpcpy (p, ") VALUES ('");
p = stpcpy (p, "INSERT INTO gncPathCache (path) VALUES ('");
p = stpcpy (p, val_str);
p = stpcpy (p, "');");
@@ -141,7 +149,7 @@ pgendGetCache (PGBackend *be,
FINISH_QUERY(be->connection);
/* and requery to get the serial number ... */
ival = pgendGetCache (be, table_name, key_name, cache_name, val_str);
ival = pgendGetCache (be, val_str);
return ival;
}
@@ -152,7 +160,7 @@ static int
pgendGetPathCache (PGBackend *be, const char *path_str)
{
int ival;
ival = pgendGetCache (be, "gncPathCache", "path", "ipath", path_str);
ival = pgendGetCache (be, path_str);
PINFO ("cached %d for %s", ival, path_str ? path_str : "(null)");
if (0 >= ival) return ival;
@@ -160,31 +168,6 @@ pgendGetPathCache (PGBackend *be, const char *path_str)
return ival;
}
/* =========================================================== */
/* given a string, return the corresponding int from the sql db. */
static int
pgendGetGUIDCacheIDStr (PGBackend *be, const char *guid_str)
{
int ival;
ival = pgendGetCache (be, "gncGUIDCache", "guid", "iguid", guid_str);
PINFO ("cached %d for %s", ival, guid_str ? guid_str : "(null)");
return ival;
}
/* =========================================================== */
/* given a guid, return the corresponding int from the sql db. */
static int
pgendGetGUIDCacheID (PGBackend *be, const GUID *guid)
{
char guid_str[GUID_ENCODING_LENGTH+1];
if (!be || !guid_str) return 0;
guid_to_string_buff (guid, guid_str);
return pgendGetGUIDCacheIDStr (be, guid_str);
}
/* =========================================================== */
/* storage of the kvp data to the database is done with the aid
* of a traversal callback. The store_cb() routine is the callback.
@@ -326,16 +309,12 @@ store_cb (const char *key, kvp_value *val, gpointer p)
}
void
pgendKVPStore (PGBackend *be, const GUID *guid, kvp_frame *kf)
pgendKVPStore (PGBackend *be, guint32 iguid, kvp_frame *kf)
{
store_data_t cb_data;
int iguid;
if (!be || !guid || !kf) return;
if (!be || 0 == iguid || !kf) return;
ENTER (" ");
iguid = pgendGetGUIDCacheID (be, guid);
if (0 == iguid) return;
cb_data.be = be;
cb_data.iguid = iguid;
cb_data.path = "";
@@ -494,13 +473,12 @@ count_handler (PGBackend *be, PGresult *result, int j, gpointer data)
kvp_frame *
pgendKVPFetch (PGBackend *be, const GUID *guid, kvp_frame *kf)
pgendKVPFetch (PGBackend *be, guint32 iguid, kvp_frame *kf)
{
char * p;
char iguid_str[40];
int iguid = 0;
int count = 0;
if (!be || !guid) return kf;
if (!be || 0 == iguid) return kf;
ENTER (" ");
@@ -508,19 +486,8 @@ pgendKVPFetch (PGBackend *be, const GUID *guid, kvp_frame *kf)
pgendKVPInit (be);
/* get the effective iguid for this object */
iguid = pgendGetGUIDCacheID (be, guid);
if (0 == iguid) return kf;
snprintf (iguid_str, 40, "%d;", iguid);
/* save on some sql queries by avoiding kvp data fetches when
* there is no data */
p = be->buff; *p = 0;
p = stpcpy (p, "SELECT count(*) FROM gncKVPValue WHERE iguid=");
p = stpcpy (p, iguid_str);
SEND_QUERY (be,be->buff, kf);
pgendGetResults (be, count_handler, &count);
if (0 == count) return kf;
/* now troll the individual tables for data */
GET_KVP(int64);
GET_KVP(dbl);
@@ -545,15 +512,11 @@ pgendKVPFetch (PGBackend *be, const GUID *guid, kvp_frame *kf)
}
void
pgendKVPDeleteStr (PGBackend *be, const char *guid)
pgendKVPDelete (PGBackend *be, guint32 iguid)
{
char iguid_str[80], sess_str[80];
char * p;
int iguid = 0;
if (!be || !guid) return;
iguid = pgendGetGUIDCacheIDStr (be, guid);
if (0 == iguid) return;
if (!be || 0 == iguid) return;
sprintf (iguid_str, "%d;\n", iguid);
guid_to_string_buff (be->sessionGuid, sess_str);
@@ -589,16 +552,4 @@ pgendKVPDeleteStr (PGBackend *be, const char *guid)
}
/* =========================================================== */
void
pgendKVPDelete (PGBackend *be, const GUID *guid)
{
char guid_str[33];
if (!be || !guid) return;
guid_to_string_buff (guid, guid_str);
return pgendKVPDeleteStr (be, guid_str);
}
/* =========================== END OF FILE ===================== */
+11 -6
View File
@@ -47,7 +47,7 @@ void pgendKVPInit (PGBackend *);
/* The pgendKVPStore() routine copies the contents of the kvp-frame
* to the SQL database, associating the root of the kvp-frame
* with the indicated GUID.
* with the indicated GUID cache index 'iguid'.
* (Note that currently it does not delete excess kvp data.
* That is, if the database has more kvp data in it than
* what was passed to this routine, then it does not delete
@@ -56,15 +56,20 @@ void pgendKVPInit (PGBackend *);
*
* The pgendKVPDelete() and pgendKVPDeleteStr() routines delete
* all kvp data in the database associated with the indicated
* GUID.
* GUID cache index 'iguid'.
*
* The pgendKVPFetch() routine pulls kvp data out of the database.
*
* The pgendNewGUIDidx() routine generates a new number suitable for
* use as a GUID cache index.
*/
void pgendKVPStore (PGBackend *, const GUID *, kvp_frame *);
void pgendKVPDelete (PGBackend *, const GUID *);
void pgendKVPDeleteStr (PGBackend *, const char *guid_string);
void pgendKVPStore (PGBackend *, guint32 iguid, kvp_frame *);
void pgendKVPDelete (PGBackend *, guint32 iguid);
kvp_frame * pgendKVPFetch (PGBackend *, guint32 iguid, kvp_frame *);
guint32 pgendNewGUIDidx (PGBackend *be);
kvp_frame * pgendKVPFetch (PGBackend *, const GUID *, kvp_frame *);
#endif /* KVP_SQL_H */
+6 -3
View File
@@ -33,7 +33,8 @@ CREATE TABLE gncAccountTrail (
description TEXT,
type TEXT NOT NULL,
commodity TEXT NOT NULL CHECK (commodity <>''),
version INT4 NOT NULL
version INT4 NOT NULL,
iguid INT4 DEFAULT 0
) INHERITS (gncAuditTrail);
CREATE INDEX gncAccountTrail_account_idx ON gncAccountTrail (accountGuid);
@@ -58,7 +59,8 @@ CREATE TABLE gncEntryTrail (
reconciled CHAR DEFAULT 'n',
date_reconciled DATETIME,
amount INT8 DEFAULT '0',
value INT8 DEFAULT '0'
value INT8 DEFAULT '0',
iguid INT4 DEFAULT 0
) INHERITS (gncAuditTrail);
CREATE INDEX gncEntryTrail_entry_idx ON gncEntryTrail (entryGuid);
@@ -85,7 +87,8 @@ CREATE TABLE gncTransactionTrail (
num TEXT,
description TEXT,
currency TEXT NOT NULL CHECK (currency <> ''),
version INT4 NOT NULL
version INT4 NOT NULL,
iguid INT4 DEFAULT 0
) INHERITS (gncAuditTrail);
CREATE INDEX gncTransactionTrail_trans_idx ON gncTransactionTrail (transGuid);
+17 -9
View File
@@ -22,6 +22,16 @@
-- Copyright (C) 2000, 2001 Linas Vepstas
--
CREATE TABLE gncVersion (
major INT NOT NULL,
minor INT NOT NULL,
rev INT DEFAULT '0',
name TEXT UNIQUE NOT NULL CHECK (name <> ''),
date DATETIME DEFAULT 'NOW'
);
INSERT INTO gncVersion (major,minor,rev,name) VALUES (1,0,0,'Version Table');
INSERT INTO gncVersion (major,minor,rev,name) VALUES (1,1,1,'iGUID in Main Tables');
-- Commodity structure
-- Store currency, security types. Namespace includes
-- ISO4217 for currencies, NASDAQ, AMEX, NYSE, EUREX for
@@ -48,7 +58,8 @@ CREATE TABLE gncAccount (
description TEXT,
type TEXT NOT NULL,
commodity TEXT NOT NULL CHECK (commodity <>''),
version INT4 NOT NULL
version INT4 NOT NULL,
iguid INT4 DEFAULT 0
);
-- CREATE INDEX gncAccount_pg_idx ON gncAccount (parentGuid);
@@ -61,7 +72,8 @@ CREATE TABLE gncTransaction (
num TEXT,
description TEXT,
currency TEXT NOT NULL CHECK (currency <> ''),
version INT4 NOT NULL
version INT4 NOT NULL,
iguid INT4 DEFAULT 0
);
CREATE INDEX gncTransaction_posted_idx ON gncTransaction (date_posted);
@@ -78,7 +90,8 @@ CREATE TABLE gncEntry (
reconciled CHAR DEFAULT 'n',
date_reconciled DATETIME,
amount INT8 DEFAULT '0',
value INT8 DEFAULT '0'
value INT8 DEFAULT '0',
iguid INT4 DEFAULT 0
);
CREATE INDEX gncEntry_acc_idx ON gncEntry (accountGuid);
@@ -151,12 +164,7 @@ CREATE TABLE gncPathCache (
path TEXT
);
CREATE TABLE gncGUIDCache (
iguid SERIAL PRIMARY KEY,
guid CHAR(32) UNIQUE NOT NULL
);
CREATE INDEX gncGUIDCache_guid_idx ON gncGUIDCache (guid);
CREATE SEQUENCE gnc_iguid_seq START 1;
CREATE TABLE gncKVPvalue (
iguid INT4,
+3
View File
@@ -12,6 +12,7 @@ define(`account', `gncAccount, Account, Account, a,
type, , char *, xaccAccountTypeEnumAsString(xaccAccountGetType(ptr)),
commodity, , char *, gnc_commodity_get_unique_name(xaccAccountGetCommodity(ptr)),
version, , int32, xaccAccountGetVersion(ptr),
iguid, , int32, ptr->idata,
parentGUID, , GUID *, xaccAccountGetGUID(xaccAccountGetParentAccount(ptr)),
accountGUID, KEY, GUID *, xaccAccountGetGUID(ptr),
')
@@ -25,6 +26,7 @@ define(`split', `gncEntry, Split, Split, e,
date_reconciled, , Timespec, xaccSplitRetDateReconciledTS(ptr),
amount, , int64, gnc_numeric_num(xaccSplitGetAmount(ptr)),
value, , int64, gnc_numeric_num(xaccSplitGetValue(ptr)),
iguid, , int32, ptr->idata,
entryGUID, KEY, GUID *, xaccSplitGetGUID(ptr),
')
@@ -43,6 +45,7 @@ define(`transaction', `gncTransaction, Transaction, Transaction, t,
date_entered, , Timespec, xaccTransRetDateEnteredTS(ptr),
date_posted, , Timespec, xaccTransRetDatePostedTS(ptr),
version, , int32, xaccTransGetVersion(ptr),
iguid, , int32, ptr->idata,
transGUID, KEY, GUID *, xaccTransGetGUID(ptr),
')
+29 -7
View File
@@ -166,7 +166,11 @@ pgendStoreTransactionNoLock (PGBackend *be, Transaction *trans,
/* destroy any associated kvp data as well */
for (node=deletelist; node; node=node->next)
{
pgendKVPDeleteStr (be, (char *)(node->data));
Split *s;
GUID guid;
string_to_guid ((char *)(node->data), &guid);
s = xaccSplitLookup(&guid);
pgendKVPDelete (be, s->idata);
g_free (node->data);
}
}
@@ -179,11 +183,21 @@ pgendStoreTransactionNoLock (PGBackend *be, Transaction *trans,
for (node=start; node; node=node->next)
{
Split * s = node->data;
if ((0 == s->idata) &&
(FALSE == kvp_frame_is_empty (xaccSplitGetSlots(s))))
{
s->idata = pgendNewGUIDidx(be);
}
pgendPutOneSplitOnly (be, s);
pgendKVPStore (be, &(s->guid), s->kvp_data);
if (s->idata) { pgendKVPStore (be, s->idata, s->kvp_data); }
}
if ((0 == trans->idata) &&
(FALSE == kvp_frame_is_empty (xaccTransGetSlots(trans))))
{
trans->idata = pgendNewGUIDidx(be);
}
pgendPutOneTransactionOnly (be, trans);
pgendKVPStore (be, &(trans->guid), trans->kvp_data);
if (trans->idata) { pgendKVPStore (be, trans->idata, trans->kvp_data); }
}
else
{
@@ -215,9 +229,9 @@ pgendStoreTransactionNoLock (PGBackend *be, Transaction *trans,
for (node=start; node; node=node->next)
{
Split * s = node->data;
pgendKVPDelete (be, &(s->guid));
if (0 != s->idata) pgendKVPDelete (be, s->idata);
}
pgendKVPDelete (be, &(trans->guid));
if (0 != trans->idata) pgendKVPDelete (be, trans->idata);
}
LEAVE(" ");
@@ -405,6 +419,7 @@ pgendCopySplitsToEngine (PGBackend *be, Transaction *trans)
xaccSplitSetDateReconciledTS (s, &ts);
xaccSplitSetReconcile (s, (DB_GET_VAL("reconciled", j))[0]);
s->idata = atoi(DB_GET_VAL("iguid",j));
/* --------------------------------------------- */
/* next, find the account that this split goes into */
@@ -617,6 +632,7 @@ pgendCopyTransactionToEngine (PGBackend *be, const GUID *trans_guid)
xaccTransSetVersion (trans, atoi(DB_GET_VAL("version",j)));
currency = gnc_string_to_commodity (DB_GET_VAL("currency",j));
xaccTransSetCurrency (trans, currency);
trans->idata = atoi(DB_GET_VAL("iguid",j));
}
}
PQclear (result);
@@ -644,13 +660,19 @@ pgendCopyTransactionToEngine (PGBackend *be, const GUID *trans_guid)
/* ------------------------------------------------- */
/* restore any kvp data associated with the transaction and splits */
trans->kvp_data = pgendKVPFetch (be, &(trans->guid), trans->kvp_data);
if (0 != trans->idata)
{
trans->kvp_data = pgendKVPFetch (be, trans->idata, trans->kvp_data);
}
engine_splits = xaccTransGetSplitList(trans);
for (node = engine_splits; node; node=node->next)
{
Split *s = node->data;
s->kvp_data = pgendKVPFetch (be, &(s->guid), s->kvp_data);
if (0 != s->idata)
{
s->kvp_data = pgendKVPFetch (be, s->idata, s->kvp_data);
}
}
/* ------------------------------------------------- */
+10 -5
View File
@@ -99,6 +99,7 @@ get_mass_trans_cb (PGBackend *be, PGresult *result, int j, gpointer data)
ts = gnc_iso8601_to_timespec_local (DB_GET_VAL("date_entered",j));
xaccTransSetDateEnteredTS (trans, &ts);
xaccTransSetVersion (trans, atoi(DB_GET_VAL("version",j)));
trans->idata = atoi (DB_GET_VAL("iguid",j));
currency = gnc_string_to_commodity (DB_GET_VAL("currency",j));
trans_frac = gnc_commodity_get_fraction (currency);
@@ -150,6 +151,7 @@ get_mass_entry_cb (PGBackend *be, PGresult *result, int j, gpointer data)
xaccSplitSetDateReconciledTS (s, &ts);
xaccSplitSetReconcile (s, (DB_GET_VAL("reconciled", j))[0]);
s->idata = atoi (DB_GET_VAL("iguid",j));
guid = nullguid; /* just in case the read fails ... */
string_to_guid (DB_GET_VAL("transGUID",j), &guid);
@@ -223,9 +225,6 @@ pgendGetMassTransactions (PGBackend *be, AccountGroup *grp)
SEND_QUERY (be, "SELECT * FROM gncEntry;", );
pgendGetResults (be, get_mass_entry_cb, NULL);
/* hack alert !!!! not restoring transaction/split slots for now !!!! */
/* this has a huge sucking sound, fix later!! */
xaction_list = NULL;
for (node=xaction_list; node; node=node->next)
{
Transaction *trans = (Transaction *)node->data;
@@ -236,13 +235,19 @@ pgendGetMassTransactions (PGBackend *be, AccountGroup *grp)
* We won't do this en-mass, as there currently seems to be no
* performance advantage to doing so */
trans->kvp_data = pgendKVPFetch (be, &(trans->guid), trans->kvp_data);
if (trans->idata)
{
trans->kvp_data = pgendKVPFetch (be, trans->idata, trans->kvp_data);
}
splits = xaccTransGetSplitList(trans);
for (snode = splits; snode; snode=snode->next)
{
Split *s = snode->data;
s->kvp_data = pgendKVPFetch (be, &(s->guid), s->kvp_data);
if (s->idata)
{
s->kvp_data = pgendKVPFetch (be, s->idata, s->kvp_data);
}
}
/* ------------------------------------------------- */
+4 -3
View File
@@ -73,7 +73,8 @@ pgendVersionTable (PGBackend *be)
" major INT NOT NULL,\n"
" minor INT NOT NULL,\n"
" rev INT DEFAULT '0',\n"
" name TEXT UNIQUE NOT NULL CHECK (name <> '')\n"
" name TEXT UNIQUE NOT NULL CHECK (name <> ''),\n"
" date DATETIME DEFAULT 'NOW' \n"
");\n"
"INSERT INTO gncVersion (major,minor,rev,name) VALUES \n"
" (1,0,0,'Version Table');";
@@ -206,6 +207,7 @@ put_iguid_in_tables (PGBackend *be)
sprintf(buff, "CREATE SEQUENCE gnc_iguid_seq START %d;", iguid);
SEND_QUERY (be,buff, );
FINISH_QUERY(be->connection);
p = "DROP TABLE gncGUIDCache;";
"INSERT INTO gncVersion (major,minor,rev,name) VALUES \n"
@@ -261,8 +263,7 @@ pgendUpgradeDB (PGBackend *be)
/* version 1.1.0 add iguids to transaction and entry tables */
if (1> vers.minor)
{
// put_iguid_in_tables(be);
PWARN (" pretending to upgrade database !xxxxxxxxxxxxxxxx\nm");
put_iguid_in_tables(be);
}
}