diff --git a/src/engine/sql/Makefile.am b/src/engine/sql/Makefile.am index 1bf056a424..22f997fdfd 100644 --- a/src/engine/sql/Makefile.am +++ b/src/engine/sql/Makefile.am @@ -2,9 +2,12 @@ # Build the postgres backend as its own loadable shared object. lib_LTLIBRARIES = libgnc_postgres.la +libgnc_postgres_la_LDFLAGS = -version-info 5:2:5 + libgnc_postgres_la_SOURCES = \ builder.c \ + checkpoint.c \ gncquery.c \ PostgresBackend.c diff --git a/src/engine/sql/PostgresBackend.c b/src/engine/sql/PostgresBackend.c index 04ddc7cbac..621e61ffd2 100644 --- a/src/engine/sql/PostgresBackend.c +++ b/src/engine/sql/PostgresBackend.c @@ -3,9 +3,10 @@ * PostgressBackend.c * * FUNCTION: - * Implements the callbacks for the postgress backend. - * this is code kinda usually works. - * it needs review and design checking + * Implements the callbacks for the Postgres backend. + * The SINGLE modes mostly work and are mostly feature complete. + * The multi-user modes are mostly in disrepair, and marginally + * functional. * * HISTORY: * Copyright (c) 2000, 2001 Linas Vepstas @@ -40,6 +41,8 @@ #include "gncquery.h" #include "PostgresBackend.h" +#include "putil.h" + static short module = MOD_BACKEND; static void pgendDisable (PGBackend *be); @@ -55,206 +58,6 @@ static const char * pgendSessionGetMode (PGBackend *be); */ #define QBUFSIZE 16350 - -/* hack alert -- calling PQFinish() is quite harsh, since all - * subsequent sql queries will fail. On the other hand, killing - * anything that follows *is* a way of minimizing data corruption - * due to subsequent mishaps ... so anyway, error handling in these - * routines needs to be rethought. - */ - -/* ============================================================= */ -/* The SEND_QUERY macro sends the sql statement off to the server. - * It performs a minimal check to see that the send succeeded. - */ - -#define SEND_QUERY(be,buff,retval) \ -{ \ - int rc; \ - rc = PQsendQuery (be->connection, buff); \ - if (!rc) \ - { \ - /* hack alert -- we need kinder, gentler error handling */\ - PERR("send query failed:\n" \ - "\t%s", PQerrorMessage(be->connection)); \ - PQfinish (be->connection); \ - xaccBackendSetError (&be->be, ERR_SQL_SEND_QUERY_FAILED); \ - return retval; \ - } \ -} - -/* --------------------------------------------------------------- */ -/* The FINISH_QUERY macro makes sure that the previously sent - * query complete with no errors. It assumes that the query - * is does not produce any results; if it did, those results are - * discarded (only error conditions are checked for). - */ - -#define FINISH_QUERY(conn) \ -{ \ - int i=0; \ - PGresult *result; \ - /* complete/commit the transaction, check the status */ \ - do { \ - ExecStatusType status; \ - result = PQgetResult((conn)); \ - if (!result) break; \ - PINFO ("clearing result %d", i); \ - status = PQresultStatus(result); \ - if (PGRES_COMMAND_OK != status) { \ - PERR("finish query failed:\n" \ - "\t%s", PQerrorMessage((conn))); \ - PQclear(result); \ - PQfinish ((conn)); \ - xaccBackendSetError (&be->be, ERR_SQL_FINISH_QUERY_FAILED); \ - } \ - PQclear(result); \ - i++; \ - } while (result); \ -} - -/* --------------------------------------------------------------- */ -/* The GET_RESULTS macro grabs the result of an pgSQL query off the - * wire, and makes sure that no errors occured. Results are left - * in the result buffer. - */ -#define GET_RESULTS(conn,result) \ -{ \ - ExecStatusType status; \ - result = PQgetResult (conn); \ - if (!result) break; \ - status = PQresultStatus(result); \ - if ((PGRES_COMMAND_OK != status) && \ - (PGRES_TUPLES_OK != status)) \ - { \ - PERR("failed to get result to query:\n" \ - "\t%s", PQerrorMessage((conn))); \ - PQclear (result); \ - PQfinish (conn); \ - xaccBackendSetError (&be->be, ERR_SQL_GET_RESULT_FAILED); \ - break; \ - } \ -} - -/* --------------------------------------------------------------- */ -/* The IF_ONE_ROW macro counts the number of rows returned by - * a query, reports an error if there is more than one row, and - * conditionally executes a block for the first row. - */ - -#define IF_ONE_ROW(result,nrows,loopcounter) \ - { \ - int ncols = PQnfields (result); \ - nrows += PQntuples (result); \ - PINFO ("query result %d has %d rows and %d cols", \ - loopcounter, nrows, ncols); \ - } \ - if (1 < nrows) { \ - PERR ("unexpected duplicate records"); \ - xaccBackendSetError (&be->be, ERR_SQL_CORRUPT_DB); \ - break; \ - } else if (1 == nrows) - -/* --------------------------------------------------------------- */ -/* Some utility macros for comparing values returned from the - * database to values in the engine structs. These macros - * all take three arguments: - * -- sqlname -- input -- the name of the field in the sql table - * -- fun -- input -- a subroutine returning a value - * -- ndiffs -- input/output -- integer, incremented if the - * value ofthe field and the value returned by - * the subroutine differ. - * - * The different macros compare different field types. - */ - -#define DB_GET_VAL(str,n) (PQgetvalue (result, n, PQfnumber (result, str))) - -/* compare string types. null strings and emty strings are - * considered to be equal */ -#define COMP_STR(sqlname,fun,ndiffs) { \ - if (null_strcmp (DB_GET_VAL(sqlname,0),fun)) { \ - PINFO("mis-match: %s sql='%s', eng='%s'", sqlname, \ - DB_GET_VAL (sqlname,0), fun); \ - ndiffs++; \ - } \ -} - -/* compare guids */ -#define COMP_GUID(sqlname,fun, ndiffs) { \ - char guid_str[GUID_ENCODING_LENGTH+1]; \ - guid_to_string_buff(fun, guid_str); \ - if (null_strcmp (DB_GET_VAL(sqlname,0),guid_str)) { \ - PINFO("mis-match: %s sql='%s', eng='%s'", sqlname, \ - DB_GET_VAL(sqlname,0), guid_str); \ - ndiffs++; \ - } \ -} - -/* comapre one char only */ -#define COMP_CHAR(sqlname,fun, ndiffs) { \ - if (tolower((DB_GET_VAL(sqlname,0))[0]) != tolower(fun)) { \ - PINFO("mis-match: %s sql=%c eng=%c", sqlname, \ - tolower((DB_GET_VAL(sqlname,0))[0]), tolower(fun)); \ - ndiffs++; \ - } \ -} - -/* Compare dates. - * Assumes the datestring is in ISO-8601 format - * i.e. looks like 1998-07-17 11:00:00.68-05 - * hack-alert doesn't compare nano-seconds .. - * this is intentional, its because I suspect - * the sql db round nanoseconds off ... - */ -#define COMP_DATE(sqlname,fun,ndiffs) { \ - Timespec eng_time = fun; \ - Timespec sql_time = gnc_iso8601_to_timespec_local( \ - DB_GET_VAL(sqlname,0)); \ - if (eng_time.tv_sec != sql_time.tv_sec) { \ - char buff[80]; \ - gnc_timespec_to_iso8601_buff(eng_time, buff); \ - PINFO("mis-match: %s sql='%s' eng=%s", sqlname, \ - DB_GET_VAL(sqlname,0), buff); \ - ndiffs++; \ - } \ -} - -/* Compare the date of last modification. - * This is a special date comp to make the m4 macros simpler. - */ -#define COMP_NOW(sqlname,fun,ndiffs) { \ - Timespec eng_time = xaccTransRetDateEnteredTS(ptr); \ - Timespec sql_time = gnc_iso8601_to_timespec_local( \ - DB_GET_VAL(sqlname,0)); \ - if (eng_time.tv_sec != sql_time.tv_sec) { \ - char buff[80]; \ - gnc_timespec_to_iso8601_buff(eng_time, buff); \ - PINFO("mis-match: %s sql='%s' eng=%s", sqlname, \ - DB_GET_VAL(sqlname,0), buff); \ - ndiffs++; \ - } \ -} - - -/* Compare long-long integers */ -#define COMP_INT64(sqlname,fun,ndiffs) { \ - if (atoll (DB_GET_VAL(sqlname,0)) != fun) { \ - PINFO("mis-match: %s sql='%s', eng='%lld'", sqlname, \ - DB_GET_VAL (sqlname,0), fun); \ - ndiffs++; \ - } \ -} - -/* compare 32-bit ints */ -#define COMP_INT32(sqlname,fun,ndiffs) { \ - if (atol (DB_GET_VAL(sqlname,0)) != fun) { \ - PINFO("mis-match: %s sql='%s', eng='%d'", sqlname, \ - DB_GET_VAL (sqlname,0), fun); \ - ndiffs++; \ - } \ -} - /* ============================================================= */ /* misc bogus utility routines */ @@ -344,7 +147,7 @@ pgendStoreAccountNoLock (PGBackend *be, Account *acct, const gnc_commodity *com; if (!be || !acct) return; - if (FALSE == acct->core_dirty) return; + if ((FALSE == do_mark) && (FALSE == acct->core_dirty)) return; ENTER ("acct=%p, mark=%d", acct, do_mark); @@ -364,7 +167,7 @@ pgendStoreAccountNoLock (PGBackend *be, Account *acct, /* make sure the account's commodity is in the commodity table */ /* hack alert -- it would be more efficient to do this elsewhere, - * and not here. */ + * and not here. Or put a mark on it ... */ com = xaccAccountGetCommodity (acct); pgendPutOneCommodityOnly (be, (gnc_commodity *) com); @@ -382,11 +185,10 @@ static void pgendStoreTransactionNoLock (PGBackend *be, Transaction *trans, gboolean do_mark) { - GUID nullguid = *(xaccGUIDNULL()); - GList *deletelist=NULL, *node; + GList *start, *deletelist=NULL, *node; PGresult *result; char * p; - int i, nrows, nsplits; + int i, nrows; if (!be || !trans) return; ENTER ("trans=%p, mark=%d", trans, do_mark); @@ -417,7 +219,6 @@ pgendStoreTransactionNoLock (PGBackend *be, Transaction *trans, { GUID guid = nullguid; string_to_guid (DB_GET_VAL ("entryGuid", j), &guid); - /* If the database has splits that the engine doesn't, * collect 'em up & we'll have to delete em */ if (NULL == xaccLookupEntity (&guid, GNC_ID_SPLIT)) @@ -433,41 +234,44 @@ pgendStoreTransactionNoLock (PGBackend *be, Transaction *trans, /* delete those that don't belong */ + p = be->buff; *p = 0; for (node=deletelist; node; node=node->next) { - p = be->buff; *p = 0; p = stpcpy (p, "DELETE FROM gncEntry WHERE entryGuid='"); p = stpcpy (p, node->data); - p = stpcpy (p, "';"); + p = stpcpy (p, "';\n"); + g_free (node->data); + } + if (p != be->buff) + { + PINFO ("%s", be->buff); SEND_QUERY (be,be->buff, ); FINISH_QUERY(be->connection); - g_free (node->data); } /* Update the rest */ - nsplits = xaccTransCountSplits (trans); + start = xaccTransGetSplitList(trans); - if ((nsplits) && !(trans->open & BEING_DESTROYED)) + if ((start) && !(trans->open & BEING_DESTROYED)) { - for (i=0; inext) + { + Split * s = node->data; pgendPutOneSplitOnly (be, s); } pgendPutOneTransactionOnly (be, trans); } else { - for (i=0; ibuff; *p = 0; + p = be->buff; *p = 0; + for (node=start; node; node=node->next) + { + Split * s = node->data; p = stpcpy (p, "DELETE FROM gncEntry WHERE entryGuid='"); p = guid_to_string_buff (xaccSplitGetGUID(s), p); - p = stpcpy (p, "';"); - PINFO ("%s\n", be->buff); - SEND_QUERY (be,be->buff, ); - FINISH_QUERY(be->connection); + p = stpcpy (p, "';\n"); } - p = be->buff; *p = 0; + p = be->buff; p = stpcpy (p, "DELETE FROM gncTransaction WHERE transGuid='"); p = guid_to_string_buff (xaccTransGetGUID(trans), p); p = stpcpy (p, "';"); @@ -520,17 +324,17 @@ static void pgendStoreGroupNoLock (PGBackend *be, AccountGroup *grp, gboolean do_mark) { - int i, nacc; + GList *start, *node; if (!be || !grp) return; ENTER("grp=%p mark=%d", grp, do_mark); /* walk the account tree, and store subaccounts */ - nacc = xaccGroupGetNumAccounts(grp); - - for (i=0; inext) + { AccountGroup *subgrp; - Account *acc = xaccGroupGetAccount(grp, i); + Account *acc = node->data; pgendStoreAccountNoLock (be, acc, do_mark); @@ -550,6 +354,7 @@ pgendStoreGroup (PGBackend *be, AccountGroup *grp) if (!be || !grp) return; /* lock it up so that we store atomically */ +/* hack alert ---- we need to lock a bunch of tables, right??!! */ bufp = "BEGIN;"; SEND_QUERY (be,bufp, ); FINISH_QUERY(be->connection); @@ -575,289 +380,6 @@ pgendStoreGroup (PGBackend *be, AccountGroup *grp) LEAVE(" "); } -/* ============================================================= */ -/* recompute *all* checkpoints for the account */ - -static void -pgendAccountRecomputeAllCheckpoints (PGBackend *be, const GUID *acct_guid) -{ - Timespec this_ts, prev_ts; - GMemChunk *chunk; - GList *node, *checkpoints = NULL; - PGresult *result; - Checkpoint *bp; - char *p; - int i, nrows, nsplits; - Account *acc; - const char *commodity_name; - - if (!be) return; - ENTER("be=%p", be); - - acc = xaccLookupEntity (acct_guid, GNC_ID_ACCOUNT); - commodity_name = gnc_commodity_get_unique_name (xaccAccountGetCommodity(acc)); - - chunk = g_mem_chunk_create (Checkpoint, 300, G_ALLOC_ONLY); - - /* prevent others from inserting any splits while we recompute - * the checkpoints. (hack alert -verify that this is the correct - * lock) */ - p = "BEGIN WORK; " - "LOCK TABLE gncEntry IN SHARE MODE; " - "LOCK TABLE gncCheckpoint IN ACCESS EXCLUSIVE MODE; "; - SEND_QUERY (be,p, ); - FINISH_QUERY(be->connection); - - /* Blow all the old checkpoints for this account out of the water. - * This should help ensure against accidental corruption. - */ - p = be->buff; *p = 0; - p = stpcpy (p, "DELETE FROM gncCheckpoint WHERE accountGuid='"); - p = guid_to_string_buff (acct_guid, p); - p = stpcpy (p, "';"); - SEND_QUERY (be,be->buff, ); - FINISH_QUERY(be->connection); - - /* and now, fetch *all* of the splits in this account */ - p = be->buff; *p = 0; - p = stpcpy (p, "SELECT gncEntry.amountNum AS amountNum, " - " gncEntry.reconciled AS reconciled," - " gncTransaction.date_posted AS date_posted " - "FROM gncEntry, gncTransaction " - "WHERE gncEntry.transGuid = gncTransaction.transGuid " - "AND accountGuid='"); - p = guid_to_string_buff (acct_guid, p); - p = stpcpy (p, "' ORDER BY gncTransaction.date_posted ASC;"); - SEND_QUERY (be,be->buff, ); - - /* malloc a new checkpoint, set it to the dawn of AD time ... */ - bp = g_chunk_new0 (Checkpoint, chunk); - checkpoints = g_list_prepend (checkpoints, bp); - this_ts = gnc_iso8601_to_timespec_local ("1970-04-15 08:35:46.00"); - bp->datetime = this_ts; - bp->account_guid = acct_guid; - bp->commodity = commodity_name; - - /* malloc a new checkpoint ... */ - nsplits = 0; - bp = g_chunk_new0 (Checkpoint, chunk); - checkpoints = g_list_prepend (checkpoints, bp); - bp->account_guid = acct_guid; - bp->commodity = commodity_name; - - /* start adding up balances */ - i=0; nrows=0; - do { - GET_RESULTS (be->connection, result); - { - int j, jrows; - int ncols = PQnfields (result); - jrows = PQntuples (result); - nrows += jrows; - PINFO ("query result %d has %d rows and %d cols", - i, nrows, ncols); - - for (j=0; jdatetime = this_ts; - - /* and now, build a new checkpoint */ - nsplits = 0; - next_bp = g_chunk_new0 (Checkpoint, chunk); - checkpoints = g_list_prepend (checkpoints, next_bp); - *next_bp = *bp; - bp = next_bp; - bp->account_guid = acct_guid; - bp->commodity = commodity_name; - } - nsplits ++; - - /* accumulate balances */ - amt = atoll (DB_GET_VAL("amountNum",j)); - recn = (DB_GET_VAL("reconciled",j))[0]; - bp->balance += amt; - if (NREC != recn) - { - bp->cleared_balance += amt; - } - if (YREC == recn) - { - bp->reconciled_balance += amt; - } - - } - } - - PQclear (result); - i++; - } while (result); - - /* set the timestamp on the final checkpoint, - * 8 seconds past the very last split */ - this_ts.tv_sec += 8; - bp->datetime = this_ts; - - /* now store the checkpoints */ - for (node = checkpoints; node; node = node->next) - { - bp = (Checkpoint *) node->data; - pgendStoreOneCheckpointOnly (be, bp, SQL_INSERT); - } - - g_list_free (checkpoints); - g_mem_chunk_destroy (chunk); - - p = "COMMIT WORK;"; - SEND_QUERY (be,p, ); - FINISH_QUERY(be->connection); - -} - -/* ============================================================= */ -/* recompute fresh balance checkpoints for every account */ - -static void -pgendGroupRecomputeAllCheckpoints (PGBackend *be, AccountGroup *grp) -{ - GList *acclist, *node; - - acclist = xaccGroupGetSubAccounts(grp); - for (node = acclist; node; node=node->next) - { - Account *acc = (Account *) node->data; - pgendAccountRecomputeAllCheckpoints (be, xaccAccountGetGUID(acc)); - } - g_list_free (acclist); -} - -/* ============================================================= */ -/* get checkpoint value for the account - * We find the checkpoint which matches the account and commodity, - * for the first date immediately preceeding the date. - * Then we fill in the balance fields for the returned query. - */ - -static void -pgendAccountGetCheckpoint (PGBackend *be, Checkpoint *chk) -{ - PGresult *result; - int i, nrows; - char * p; - - if (!be || !chk) return; - ENTER("be=%p", be); - - /* create the query we need */ - p = be->buff; *p = 0; - p = stpcpy (p, "SELECT balance, cleared_balance, reconciled_balance " - "FROM gncCheckpoint " - "WHERE accountGuid='"); - p = guid_to_string_buff (chk->account_guid, p); - p = stpcpy (p, "' AND commodity='"); - p = stpcpy (p, chk->commodity); - p = stpcpy (p, "' AND date_xpoint <'"); - p = gnc_timespec_to_iso8601_buff (chk->datetime, p); - p = stpcpy (p, "' ORDER BY date_xpoint DESC LIMIT 1;"); - SEND_QUERY (be,be->buff, ); - - i=0; nrows=0; - do { - GET_RESULTS (be->connection, result); - { - int j=0, jrows; - int ncols = PQnfields (result); - jrows = PQntuples (result); - nrows += jrows; - PINFO ("query result %d has %d rows and %d cols", - i, nrows, ncols); - - if (1 < nrows) - { - PERR ("excess data"); - PQclear (result); - return; - } - chk->balance = atoll(DB_GET_VAL("balance", j)); - chk->cleared_balance = atoll(DB_GET_VAL("cleared_balance", j)); - chk->reconciled_balance = atoll(DB_GET_VAL("reconciled_balance", j)); - } - - PQclear (result); - i++; - } while (result); - - LEAVE("be=%p", be); -} - -/* ============================================================= */ -/* get checkpoint value for all accounts */ - -static void -pgendGroupGetAllCheckpoints (PGBackend *be, AccountGroup*grp) -{ - Checkpoint chk; - GList *acclist, *node; - - if (!be || !grp) return; - ENTER("be=%p", be); - - chk.datetime.tv_sec = time(0); - chk.datetime.tv_nsec = 0; - - acclist = xaccGroupGetSubAccounts (grp); - - /* loop over all accounts */ - for (node=acclist; node; node=node->next) - { - Account *acc; - const gnc_commodity *com; - gint64 deno; - gnc_numeric baln; - gnc_numeric cleared_baln; - gnc_numeric reconciled_baln; - - /* setupwhat we will match for */ - acc = (Account *) node->data; - com = xaccAccountGetCommodity(acc); - chk.commodity = gnc_commodity_get_unique_name(com); - chk.account_guid = xaccAccountGetGUID (acc); - chk.balance = 0; - chk.cleared_balance = 0; - chk.reconciled_balance = 0; - - /* get the checkpoint */ - pgendAccountGetCheckpoint (be, &chk); - - /* set the account balances */ - deno = gnc_commodity_get_fraction (com); - baln = gnc_numeric_create (chk.balance, deno); - cleared_baln = gnc_numeric_create (chk.cleared_balance, deno); - reconciled_baln = gnc_numeric_create (chk.reconciled_balance, deno); - - xaccAccountSetStartingBalance (acc, baln, - cleared_baln, reconciled_baln); - } - - g_list_free (acclist); - LEAVE("be=%p", be); -} - /* ============================================================= */ static void @@ -939,7 +461,6 @@ static gboolean pgendCopyTransactionToEngine (PGBackend *be, GUID *trans_guid) { const gnc_commodity *modity=NULL; - GUID nullguid = *(xaccGUIDNULL()); char *pbuff; Transaction *trans; PGresult *result; @@ -1284,7 +805,6 @@ IsGuidInList (GList *list, GUID *guid) static void pgendRunQueryHelper (PGBackend *be, const char *qstring) { - GUID nullguid = *(xaccGUIDNULL()); PGresult *result; int i, nrows; GList *node, *xact_list = NULL; @@ -1444,7 +964,6 @@ pgendGetAllAccounts (PGBackend *be) AccountGroup *topgrp; char * bufp; int i, nrows, iacc; - GUID nullguid = *(xaccGUIDNULL()); ENTER ("be=%p", be); if (!be) return NULL; @@ -1590,8 +1109,9 @@ pgend_trans_commit_edit (Backend * bend, Transaction * trans, Transaction * oldtrans) { + GList *start, *node; char * bufp; - int i, ndiffs, nsplits, rollback=0; + int ndiffs, rollback=0; PGBackend *be = (PGBackend *)bend; ENTER ("be=%p, trans=%p", be, trans); @@ -1626,9 +1146,10 @@ pgend_trans_commit_edit (Backend * bend, if (0 < ndiffs) rollback++; /* be sure to check the old splits as well ... */ - nsplits = xaccTransCountSplits (oldtrans); - for (i=0; inext) + { + Split * s = node->data; ndiffs = pgendCompareOneSplitOnly (be, s); if (0 < ndiffs) rollback++; } @@ -2322,6 +1843,9 @@ pgendEnable (PGBackend *be) static void pgendInit (PGBackend *be) { + /* initialize global variable */ + nullguid = *(xaccGUIDNULL()); + /* access mode */ be->session_mode = MODE_NONE; be->sessionGuid = NULL; diff --git a/src/engine/sql/PostgresBackend.h b/src/engine/sql/PostgresBackend.h index 9e70de3992..9fddd3c323 100644 --- a/src/engine/sql/PostgresBackend.h +++ b/src/engine/sql/PostgresBackend.h @@ -10,6 +10,9 @@ */ +#ifndef __POSTGRES_BACKEND_H__ +#define __POSTGRES_BACKEND_H__ + #include #include "BackendP.h" @@ -82,3 +85,32 @@ typedef struct _checkpoint { gint64 reconciled_balance; } Checkpoint; +/* -------------------------------------------------------- */ +/* the following prototypes belong in a 'checkpoint.h' file */ + +void pgendGroupRecomputeAllCheckpoints (PGBackend *be, AccountGroup *grp); +void pgendGroupGetAllCheckpoints (PGBackend *be, AccountGroup*grp); + +/* -------------------------------------------------------- */ +/* the following protypes belong in a 'autogen.h' file */ + +void pgendStoreOneAccountOnly (PGBackend *be, Account *ptr, + sqlBuild_QType update); + +void pgendStoreOneCheckpointOnly (PGBackend *be, Checkpoint *ptr, + sqlBuild_QType update); + +void pgendStoreOneCommodityOnly (PGBackend *be, gnc_commodity *ptr, + sqlBuild_QType update); + +void pgendStoreOneSessionOnly (PGBackend *be, void *ptr, + sqlBuild_QType update); + +void pgendStoreOneSplitOnly (PGBackend *be, Split *ptr, + sqlBuild_QType update); + +void pgendStoreOneTransactionOnly (PGBackend *be, Transaction *ptr, + sqlBuild_QType update); + + +#endif /* __POSTGRES_BACKEND_H__ */ diff --git a/src/engine/sql/README b/src/engine/sql/README index fa93c71316..8ee995b9f1 100644 --- a/src/engine/sql/README +++ b/src/engine/sql/README @@ -7,7 +7,7 @@ problems. -postgres install instructions +Postgres Install Instructions ----------------------------- 1) Install postgresql server, client and devel packages. 2) if installed from redhat, then running /etc/rc.d/init.d/postgresql @@ -18,6 +18,32 @@ postgres install instructions 5) as yourself (i.e. your unix login), run 'createdb gnucash' +GnuCash Build Instructions +-------------------------- +Same as usual, but you want to specify the flag '--enable-sql' i.e. +./configure --enable-sql +and then 'make'. + + + +How To Use This Thing +--------------------- +a) Open your favorite datafile in the usual fashion. +b) Click on 'Save As' +c) enter the following URL instead of a filename in the file picker: + postgres://localhost/some-name-you-pick + +The above steps will copy your data into that database. You can +then restart gnucash (or keep working) and type in the same URL +in the file open dialogs. Or try it on the commandline: + +/usr/local/bin/gnucash postgres://localhost/whatever + +Note that you *must* use 'localhost' for your hostname, and +not some other hostname. This is because anything else requires +an sql username & password, and we don't have the gui dialogs for +that yet. + To Be Done @@ -34,11 +60,14 @@ Core bugs/features that still need work: -- bug: group sync doesn't pull in newer data from the db ... (related to the 'save as' above) --- allow user to enter URL in GUI dialog +-- allow user to enter URL in GUI dialog, get GUI to remember the URL -- Implement GUI to ask user for username/password to log onto the server. +-- fix the annoying postgres:,,localhost,asdf file syntax: needs + mods to gnc-book to keep it happy about lock files & such. + To Be Done, Part II ------------------- @@ -47,10 +76,11 @@ This list only affects the multi-user and advanced/optional features. -- implement account commit edit (actually, the check&rollback part) (need to check the account version number beofer the commit happens) --- fix excessive use of account commit by engine +-- use version numbers for accounts, commodities, splits & transactions, + as this will provide a far more efficient 'compare' for + user changes. + -- provide support for more query types in gncquery.c --- optimize for quantity of SQL traffic -- there's a lot of it, - much of it probably un-needed. -- Implement logging history in the SQL server. i.e. save the old copies of stuff in log tables. Make the username part of the diff --git a/src/engine/sql/checkpoint.c b/src/engine/sql/checkpoint.c new file mode 100644 index 0000000000..a92a7d64d0 --- /dev/null +++ b/src/engine/sql/checkpoint.c @@ -0,0 +1,324 @@ +/* + * FILE: + * checkpoint.c + * + * FUNCTION: + * account balance checkpointing. + * + * HISTORY: + * Copyright (c) 2000, 2001 Linas Vepstas + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include + +#include + +#include "Account.h" +#include "AccountP.h" +#include "Backend.h" +#include "BackendP.h" +#include "Group.h" +#include "gnc-commodity.h" +// #include "gnc-engine.h" +#include "gnc-engine-util.h" +// #include "gnc-event.h" +#include "guid.h" +#include "GNCId.h" +#include "GNCIdP.h" + +#include "builder.h" +#include "PostgresBackend.h" + +#include "putil.h" + +static short module = MOD_BACKEND; + +/* ============================================================= */ +/* recompute *all* checkpoints for the account */ + +static void +pgendAccountRecomputeAllCheckpoints (PGBackend *be, const GUID *acct_guid) +{ + Timespec this_ts, prev_ts; + GMemChunk *chunk; + GList *node, *checkpoints = NULL; + PGresult *result; + Checkpoint *bp; + char *p; + int i, nrows, nsplits; + Account *acc; + const char *commodity_name; + + if (!be) return; + ENTER("be=%p", be); + + acc = xaccLookupEntity (acct_guid, GNC_ID_ACCOUNT); + commodity_name = gnc_commodity_get_unique_name (xaccAccountGetCommodity(acc)); + + chunk = g_mem_chunk_create (Checkpoint, 300, G_ALLOC_ONLY); + + /* prevent others from inserting any splits while we recompute + * the checkpoints. (hack alert -verify that this is the correct + * lock) */ + p = "BEGIN WORK; " + "LOCK TABLE gncEntry IN SHARE MODE; " + "LOCK TABLE gncCheckpoint IN ACCESS EXCLUSIVE MODE; "; + SEND_QUERY (be,p, ); + FINISH_QUERY(be->connection); + + /* Blow all the old checkpoints for this account out of the water. + * This should help ensure against accidental corruption. + */ + p = be->buff; *p = 0; + p = stpcpy (p, "DELETE FROM gncCheckpoint WHERE accountGuid='"); + p = guid_to_string_buff (acct_guid, p); + p = stpcpy (p, "';"); + SEND_QUERY (be,be->buff, ); + FINISH_QUERY(be->connection); + + /* and now, fetch *all* of the splits in this account */ + p = be->buff; *p = 0; + p = stpcpy (p, "SELECT gncEntry.amountNum AS amountNum, " + " gncEntry.reconciled AS reconciled," + " gncTransaction.date_posted AS date_posted " + "FROM gncEntry, gncTransaction " + "WHERE gncEntry.transGuid = gncTransaction.transGuid " + "AND accountGuid='"); + p = guid_to_string_buff (acct_guid, p); + p = stpcpy (p, "' ORDER BY gncTransaction.date_posted ASC;"); + SEND_QUERY (be,be->buff, ); + + /* malloc a new checkpoint, set it to the dawn of AD time ... */ + bp = g_chunk_new0 (Checkpoint, chunk); + checkpoints = g_list_prepend (checkpoints, bp); + this_ts = gnc_iso8601_to_timespec_local ("1970-04-15 08:35:46.00"); + bp->datetime = this_ts; + bp->account_guid = acct_guid; + bp->commodity = commodity_name; + + /* malloc a new checkpoint ... */ + nsplits = 0; + bp = g_chunk_new0 (Checkpoint, chunk); + checkpoints = g_list_prepend (checkpoints, bp); + bp->account_guid = acct_guid; + bp->commodity = commodity_name; + + /* start adding up balances */ + i=0; nrows=0; + do { + GET_RESULTS (be->connection, result); + { + int j, jrows; + int ncols = PQnfields (result); + jrows = PQntuples (result); + nrows += jrows; + PINFO ("query result %d has %d rows and %d cols", + i, nrows, ncols); + + for (j=0; jdatetime = this_ts; + + /* and now, build a new checkpoint */ + nsplits = 0; + next_bp = g_chunk_new0 (Checkpoint, chunk); + checkpoints = g_list_prepend (checkpoints, next_bp); + *next_bp = *bp; + bp = next_bp; + bp->account_guid = acct_guid; + bp->commodity = commodity_name; + } + nsplits ++; + + /* accumulate balances */ + amt = atoll (DB_GET_VAL("amountNum",j)); + recn = (DB_GET_VAL("reconciled",j))[0]; + bp->balance += amt; + if (NREC != recn) + { + bp->cleared_balance += amt; + } + if (YREC == recn) + { + bp->reconciled_balance += amt; + } + + } + } + + PQclear (result); + i++; + } while (result); + + /* set the timestamp on the final checkpoint, + * 8 seconds past the very last split */ + this_ts.tv_sec += 8; + bp->datetime = this_ts; + + /* now store the checkpoints */ + for (node = checkpoints; node; node = node->next) + { + bp = (Checkpoint *) node->data; + pgendStoreOneCheckpointOnly (be, bp, SQL_INSERT); + } + + g_list_free (checkpoints); + g_mem_chunk_destroy (chunk); + + p = "COMMIT WORK;"; + SEND_QUERY (be,p, ); + FINISH_QUERY(be->connection); + +} + +/* ============================================================= */ +/* recompute fresh balance checkpoints for every account */ + +void +pgendGroupRecomputeAllCheckpoints (PGBackend *be, AccountGroup *grp) +{ + GList *acclist, *node; + + acclist = xaccGroupGetSubAccounts(grp); + for (node = acclist; node; node=node->next) + { + Account *acc = (Account *) node->data; + pgendAccountRecomputeAllCheckpoints (be, xaccAccountGetGUID(acc)); + } + g_list_free (acclist); +} + +/* ============================================================= */ +/* get checkpoint value for the account + * We find the checkpoint which matches the account and commodity, + * for the first date immediately preceeding the date. + * Then we fill in the balance fields for the returned query. + */ + +static void +pgendAccountGetCheckpoint (PGBackend *be, Checkpoint *chk) +{ + PGresult *result; + int i, nrows; + char * p; + + if (!be || !chk) return; + ENTER("be=%p", be); + + /* create the query we need */ + p = be->buff; *p = 0; + p = stpcpy (p, "SELECT balance, cleared_balance, reconciled_balance " + "FROM gncCheckpoint " + "WHERE accountGuid='"); + p = guid_to_string_buff (chk->account_guid, p); + p = stpcpy (p, "' AND commodity='"); + p = stpcpy (p, chk->commodity); + p = stpcpy (p, "' AND date_xpoint <'"); + p = gnc_timespec_to_iso8601_buff (chk->datetime, p); + p = stpcpy (p, "' ORDER BY date_xpoint DESC LIMIT 1;"); + SEND_QUERY (be,be->buff, ); + + i=0; nrows=0; + do { + GET_RESULTS (be->connection, result); + { + int j=0, jrows; + int ncols = PQnfields (result); + jrows = PQntuples (result); + nrows += jrows; + PINFO ("query result %d has %d rows and %d cols", + i, nrows, ncols); + + if (1 < nrows) + { + PERR ("excess data"); + PQclear (result); + return; + } + chk->balance = atoll(DB_GET_VAL("balance", j)); + chk->cleared_balance = atoll(DB_GET_VAL("cleared_balance", j)); + chk->reconciled_balance = atoll(DB_GET_VAL("reconciled_balance", j)); + } + + PQclear (result); + i++; + } while (result); + + LEAVE("be=%p", be); +} + +/* ============================================================= */ +/* get checkpoint value for all accounts */ + +void +pgendGroupGetAllCheckpoints (PGBackend *be, AccountGroup*grp) +{ + Checkpoint chk; + GList *acclist, *node; + + if (!be || !grp) return; + ENTER("be=%p", be); + + chk.datetime.tv_sec = time(0); + chk.datetime.tv_nsec = 0; + + acclist = xaccGroupGetSubAccounts (grp); + + /* loop over all accounts */ + for (node=acclist; node; node=node->next) + { + Account *acc; + const gnc_commodity *com; + gint64 deno; + gnc_numeric baln; + gnc_numeric cleared_baln; + gnc_numeric reconciled_baln; + + /* setupwhat we will match for */ + acc = (Account *) node->data; + com = xaccAccountGetCommodity(acc); + chk.commodity = gnc_commodity_get_unique_name(com); + chk.account_guid = xaccAccountGetGUID (acc); + chk.balance = 0; + chk.cleared_balance = 0; + chk.reconciled_balance = 0; + + /* get the checkpoint */ + pgendAccountGetCheckpoint (be, &chk); + + /* set the account balances */ + deno = gnc_commodity_get_fraction (com); + baln = gnc_numeric_create (chk.balance, deno); + cleared_baln = gnc_numeric_create (chk.cleared_balance, deno); + reconciled_baln = gnc_numeric_create (chk.reconciled_balance, deno); + + xaccAccountSetStartingBalance (acc, baln, + cleared_baln, reconciled_baln); + } + + g_list_free (acclist); + LEAVE("be=%p", be); +} + +/* ======================== END OF FILE ======================== */ diff --git a/src/engine/sql/putil.h b/src/engine/sql/putil.h new file mode 100644 index 0000000000..f3f2c8c919 --- /dev/null +++ b/src/engine/sql/putil.h @@ -0,0 +1,258 @@ +/* + * FILE: + * putil.h + * + * FUNCTION: + * Postgres backend utility macros + * + * HISTORY: + * Copyright (c) 2000, 2001 Linas Vepstas + * + */ + +#ifndef __P_UTIL_H__ +#define __P_UTIL_H__ + +#include +#include +#include + +#include + +#include "Backend.h" +#include "BackendP.h" +#include "gnc-engine-util.h" +#include "guid.h" +#include "GNCId.h" + +#include "PostgresBackend.h" + + +extern GUID nullguid; + +/* hack alert -- calling PQFinish() is quite harsh, since all + * subsequent sql queries will fail. On the other hand, killing + * anything that follows *is* a way of minimizing data corruption + * due to subsequent mishaps ... so anyway, error handling in these + * routines needs to be rethought. + */ + +/* ============================================================= */ +/* The SEND_QUERY macro sends the sql statement off to the server. + * It performs a minimal check to see that the send succeeded. + */ + +#define SEND_QUERY(be,buff,retval) \ +{ \ + int rc; \ + rc = PQsendQuery (be->connection, buff); \ + if (!rc) \ + { \ + /* hack alert -- we need kinder, gentler error handling */\ + PERR("send query failed:\n" \ + "\t%s", PQerrorMessage(be->connection)); \ + PQfinish (be->connection); \ + xaccBackendSetError (&be->be, ERR_SQL_SEND_QUERY_FAILED); \ + return retval; \ + } \ +} + +/* --------------------------------------------------------------- */ +/* The FINISH_QUERY macro makes sure that the previously sent + * query complete with no errors. It assumes that the query + * is does not produce any results; if it did, those results are + * discarded (only error conditions are checked for). + */ + +#define FINISH_QUERY(conn) \ +{ \ + int i=0; \ + PGresult *result; \ + /* complete/commit the transaction, check the status */ \ + do { \ + ExecStatusType status; \ + result = PQgetResult((conn)); \ + if (!result) break; \ + PINFO ("clearing result %d", i); \ + status = PQresultStatus(result); \ + if (PGRES_COMMAND_OK != status) { \ + PERR("finish query failed:\n" \ + "\t%s", PQerrorMessage((conn))); \ + PQclear(result); \ + PQfinish ((conn)); \ + xaccBackendSetError (&be->be, ERR_SQL_FINISH_QUERY_FAILED); \ + } \ + PQclear(result); \ + i++; \ + } while (result); \ +} + +/* --------------------------------------------------------------- */ +/* The GET_RESULTS macro grabs the result of an pgSQL query off the + * wire, and makes sure that no errors occured. Results are left + * in the result buffer. + */ +#define GET_RESULTS(conn,result) \ +{ \ + ExecStatusType status; \ + result = PQgetResult (conn); \ + if (!result) break; \ + status = PQresultStatus(result); \ + if ((PGRES_COMMAND_OK != status) && \ + (PGRES_TUPLES_OK != status)) \ + { \ + PERR("failed to get result to query:\n" \ + "\t%s", PQerrorMessage((conn))); \ + PQclear (result); \ + PQfinish (conn); \ + xaccBackendSetError (&be->be, ERR_SQL_GET_RESULT_FAILED); \ + break; \ + } \ +} + +/* --------------------------------------------------------------- */ +/* The IF_ONE_ROW macro counts the number of rows returned by + * a query, reports an error if there is more than one row, and + * conditionally executes a block for the first row. + */ + +#define IF_ONE_ROW(result,nrows,loopcounter) \ + { \ + int ncols = PQnfields (result); \ + nrows += PQntuples (result); \ + PINFO ("query result %d has %d rows and %d cols", \ + loopcounter, nrows, ncols); \ + } \ + if (1 < nrows) { \ + PERR ("unexpected duplicate records"); \ + xaccBackendSetError (&be->be, ERR_SQL_CORRUPT_DB); \ + break; \ + } else if (1 == nrows) + +/* --------------------------------------------------------------- */ +/* Some utility macros for comparing values returned from the + * database to values in the engine structs. These macros + * all take three arguments: + * -- sqlname -- input -- the name of the field in the sql table + * -- fun -- input -- a subroutine returning a value + * -- ndiffs -- input/output -- integer, incremented if the + * value ofthe field and the value returned by + * the subroutine differ. + * + * The different macros compare different field types. + */ + +#define DB_GET_VAL(str,n) (PQgetvalue (result, n, PQfnumber (result, str))) + +/* Compare string types. Null strings and empty strings are + * considered to be equal */ +#define COMP_STR(sqlname,fun,ndiffs) { \ + if (null_strcmp (DB_GET_VAL(sqlname,0),fun)) { \ + PINFO("mis-match: %s sql='%s', eng='%s'", sqlname, \ + DB_GET_VAL (sqlname,0), fun); \ + ndiffs++; \ + } \ +} + +/* Compare commodities. This routine is almost identical to + * COMP_STR, except that a NULL currency from the engine + * is allowed to match any currency in the sql DB. This is + * used to facilitate deletion, where the currency has been + * nulled out .. */ +#define COMP_COMMODITY(sqlname,fun,ndiffs) { \ + const char *com = fun; \ + if (com) { \ + if (null_strcmp (DB_GET_VAL(sqlname,0),com)) { \ + PINFO("mis-match: %s sql='%s', eng='%s'", sqlname, \ + DB_GET_VAL (sqlname,0), fun); \ + ndiffs++; \ + } \ + } \ +} + +/* Compare guids. A NULL GUID from the engine is considered to + * match any value of a GUID in teh sql database. This is + * equality is used to enable deletion, where the GUID may have + * already been set to NULL in the engine, but not yet in the DB. + */ +#define COMP_GUID(sqlname,fun, ndiffs) { \ + char guid_str[GUID_ENCODING_LENGTH+1]; \ + const GUID *guid = fun; \ + if (!guid_equal (guid, &nullguid)) { \ + guid_to_string_buff(guid, guid_str); \ + if (null_strcmp (DB_GET_VAL(sqlname,0),guid_str)) { \ + PINFO("mis-match: %s sql='%s', eng='%s'", sqlname, \ + DB_GET_VAL(sqlname,0), guid_str); \ + ndiffs++; \ + } \ + } \ +} + +/* Comapre one char only */ +#define COMP_CHAR(sqlname,fun, ndiffs) { \ + if (tolower((DB_GET_VAL(sqlname,0))[0]) != tolower(fun)) { \ + PINFO("mis-match: %s sql=%c eng=%c", sqlname, \ + tolower((DB_GET_VAL(sqlname,0))[0]), tolower(fun)); \ + ndiffs++; \ + } \ +} + +/* Compare dates. + * Assumes the datestring is in ISO-8601 format + * i.e. looks like 1998-07-17 11:00:00.68-05 + * hack-alert doesn't compare nano-seconds .. + * this is intentional, its because I suspect + * the sql db round nanoseconds off ... + */ +#define COMP_DATE(sqlname,fun,ndiffs) { \ + Timespec eng_time = fun; \ + Timespec sql_time = gnc_iso8601_to_timespec_local( \ + DB_GET_VAL(sqlname,0)); \ + if (eng_time.tv_sec != sql_time.tv_sec) { \ + char buff[80]; \ + gnc_timespec_to_iso8601_buff(eng_time, buff); \ + PINFO("mis-match: %s sql='%s' eng=%s", sqlname, \ + DB_GET_VAL(sqlname,0), buff); \ + ndiffs++; \ + } \ +} + +/* Compare the date of last modification. + * This is a special date comp to make the m4 macros simpler. + */ +#define COMP_NOW(sqlname,fun,ndiffs) { \ + Timespec eng_time = xaccTransRetDateEnteredTS(ptr); \ + Timespec sql_time = gnc_iso8601_to_timespec_local( \ + DB_GET_VAL(sqlname,0)); \ + if (eng_time.tv_sec != sql_time.tv_sec) { \ + char buff[80]; \ + gnc_timespec_to_iso8601_buff(eng_time, buff); \ + PINFO("mis-match: %s sql='%s' eng=%s", sqlname, \ + DB_GET_VAL(sqlname,0), buff); \ + ndiffs++; \ + } \ +} + + +/* Compare long-long integers */ +#define COMP_INT64(sqlname,fun,ndiffs) { \ + if (atoll (DB_GET_VAL(sqlname,0)) != fun) { \ + PINFO("mis-match: %s sql='%s', eng='%lld'", sqlname, \ + DB_GET_VAL (sqlname,0), fun); \ + ndiffs++; \ + } \ +} + +/* compare 32-bit ints */ +#define COMP_INT32(sqlname,fun,ndiffs) { \ + if (atol (DB_GET_VAL(sqlname,0)) != fun) { \ + PINFO("mis-match: %s sql='%s', eng='%d'", sqlname, \ + DB_GET_VAL (sqlname,0), fun); \ + ndiffs++; \ + } \ +} + + +#endif /* __P_UTIL_H__ */ + +/* ======================== END OF FILE ======================== */ diff --git a/src/engine/sql/table.m4 b/src/engine/sql/table.m4 index a9f2ca78f0..0870889954 100644 --- a/src/engine/sql/table.m4 +++ b/src/engine/sql/table.m4 @@ -41,7 +41,7 @@ define(`split', `gncEntry, Split, Split, define(`transaction', `gncTransaction, Transaction, Transaction, num, , char *, xaccTransGetNum(ptr), description, , char *, xaccTransGetDescription(ptr), - currency, , char *, gnc_commodity_get_unique_name(xaccTransGetCurrency(ptr)), + currency, , commod, gnc_commodity_get_unique_name(xaccTransGetCurrency(ptr)), date_entered, , now, "NOW", date_posted, , Timespec, xaccTransRetDatePostedTS(ptr), transGUID, KEY, GUID *, xaccTransGetGUID(ptr), @@ -97,6 +97,7 @@ define(`sql_setter', `ifelse($2, `KEY', $2, , `ifelse($1, `char *', sqlBuild_Set_Str, $1, `now', sqlBuild_Set_Str, + $1, `commod', sqlBuild_Set_Str, $1, `int32', sqlBuild_Set_Int32, $1, `int64', sqlBuild_Set_Int64, $1, `GUID *', sqlBuild_Set_GUID, @@ -114,12 +115,15 @@ define(`set_fields', `set_fields_r(firstrec($@))') /* -------- */ /* macros to compare a query result */ +/* the commod type behaves just like a string, except it + * has its one compre function. */ define(`cmp_value', `ifelse($1, `char *', COMP_STR, $1, `now', COMP_NOW, $1, `int32', COMP_INT32, $1, `int64', COMP_INT64, $1, `GUID *', COMP_GUID, + $1, `commod', COMP_COMMODITY, $1, `Timespec', COMP_DATE, $1, `char', COMP_CHAR)') @@ -143,7 +147,7 @@ define(`store_one_only', * It just pokes the data in */ -static void +void pgendStoreOne`'func_name($@)`'Only (PGBackend *be, xacc_type($@) *ptr, sqlBuild_QType update)