megazord vs. bugs & features

now works more better.


git-svn-id: svn+ssh://svn.gnucash.org/repo/gnucash/trunk@3551 57a11ea4-9604-0410-9ed3-97b8803252fd

src/engine/sql/PostgresBackend.h

@@ -1,8 +1,12 @@
 /* 
+ * FILE:
  * PostgresBackend.h
  *
+ * FUNCTION:
  * Implements the callbacks for the postgres backend.
- * 
+ *
+ * HISTORY:
+ * Copyright (c) 2000, 2001 Linas Vepstas 
  */
 
 
@@ -11,9 +15,24 @@
 
 typedef struct _pgend PGBackend;
 
+typedef enum {
+   MODE_NONE = 0,
+   MODE_SINGLE_FILE =1,
+   MODE_SINGLE_UPDATE,
+   MODE_POLL,
+   MODE_EVENT
+} AccessMode;
+
 struct _pgend {
    Backend be;
 
+   /* snr is used only for temporarily saving hook values */
+   Backend snr;    
+
+   /* session mode */
+   AccessMode session_mode;
+   GUID *sessionGuid;
+
    /* sql query compiler */
    sqlBuilder *builder;
 
@@ -36,3 +55,30 @@ struct _pgend {
  */
 Backend * pgendNew (void);
 
+
+/* -------------------------------------------------------- */
+/* The balance checkpoint structure is used to store partial,
+ * running balances.  The balances are correct for the checkpoint
+ * date shown.  The commodity indicates what commodity the 
+ * balances are valued in (they need not be in the same 
+ * commodity as the account)
+ */
+
+/* the MIN_CHECKPOINT_COUNT value is the number of splits that
+ * each checkpoint will handle, on avergage.  30 seems like a good
+ * number.  The number of splits in a checkpoint will vary; 
+ * checkpoints can onmly occur in between entry dates, so a 
+ * bunch of entries with the same date will go into the same 
+ * checkpoint (and there might be an arbitrarily large number of these)
+ */
+#define MIN_CHECKPOINT_COUNT 3
+
+typedef struct _checkpoint {
+   const GUID *account_guid;
+   const char * commodity;
+   Timespec datetime;
+   gint64 balance;
+   gint64 cleared_balance;
+   gint64 reconciled_balance;
+} Checkpoint;
+

src/engine/sql/README

@@ -18,7 +18,7 @@ A) run the script gnc-init.sh
 B) edit gnc-book.c and #define SQLHACK
 C) edit src/Makefile.am and add  engine/sql/libgnc_postgres.la 
    and -lpq to LDADD
-D) cd src/engine/sql; m4 table.m4 > tmp.c; make
+D) cd src/engine/sql; m4 table.m4 > autogen.c; make
 E) cd ../../..; make; make install
 
 
@@ -36,6 +36,7 @@ To Bo Done
 ----------
 -- Implement acount and transaction deletion. Deleting an account or
    transaction doesn't remove it from the database.
+   (done with split delete)
 
 -- Implement logging history in the sql server.  i.e. save the old
    copies of stuff in log tables.  Make the username part of the 
@@ -44,6 +45,9 @@ To Bo Done
 -- Implement gui to ask user for username/password to log onto the
    server. 
 
+-- let all attached client receive update events via sql LISTEN/NOTIFY 
+   events.
+
 -- Implement various advanced database features, such as checking the
    user's permission to view/edit account by account ... (hmmm this
    done by the dbadmin... using sql commands... which means if user
@@ -72,10 +76,9 @@ To Bo Done
 -- finish implementing pgendAccountGroupSync
 
 -- store account balances in database.  This will be tricky ...
-   -- need to write stored proceedures to recalc split balances
+   -- split query gets hard ... 
-   -- need to modify engine and maybe gui 
 -- urlencode all strings (escape single-quote mark)
--- check multiuser operation
+-- review multiuser operation for correctness 
 -- allow user to enter url in gui dialog
 -- handle kvp frames
 -- implement account commit edit (actually, the check&rollback part)
@@ -89,5 +92,7 @@ To Bo Done
    them out.  But right now, this is brroken.  In particular, 
    the use of xaccGrouparkSaved screws up some status bits ...
 
+-- review & match up against docs at 
+   http://www.lupercalia.net/gnc-db/
 
 

src/engine/sql/builder.c

@@ -19,6 +19,7 @@
 #include <glib.h>
 #include <string.h>
 
+#include "date.h"
 #include "builder.h"
 #include "gnc-engine-util.h"
 
@@ -182,10 +183,9 @@ void
 sqlBuild_Set_GUID (sqlBuilder *b, const char *tag, const GUID *val)
 {
   if (val) {
-     char *guid;
+     char guid_str[GUID_ENCODING_LENGTH+1];
-     guid = guid_to_string(val);
+     guid_to_string_buff(val, guid_str);
-     sqlBuild_Set_Str (b, tag, guid);
+     sqlBuild_Set_Str (b, tag, guid_str);
-     g_free (guid);
   } else {
      /* if a SELECT statement is being built, then val may be null */
      sqlBuild_Set_Str (b, tag, "");
@@ -197,28 +197,8 @@ sqlBuild_Set_GUID (sqlBuilder *b, const char *tag, const GUID *val)
 void
 sqlBuild_Set_Date (sqlBuilder *b, const char *tag, Timespec ts)
 {
-  int tz_hour, tz_min;
-  char cyn;
   char buf[512];
-  time_t tmp;
+  gnc_timespec_to_iso8601_buff (ts, buf);
-  struct tm parsed;
-  tmp = ts.tv_sec;
-  localtime_r(&tmp, &parsed);
-
-  tz_hour = timezone/3600;
-  tz_min = (timezone - 3600*tz_hour)/60;
-  if (0>tz_min) { tz_min +=60; tz_hour --; }
-
-  /* we also have to print the sign by hand, to work around a bug 
-   * in the glibc 2.1.3 printf (where %+02d fails to zero-pad)
-   */
-  cyn = '+';
-  if (0>tz_hour) { cyn = '-'; tz_hour = -tz_hour; }
-  
-  snprintf (buf, 512, "%4d-%02d-%02d %02d:%02d:%02d.%06ld %c%02d%02d",
-       parsed.tm_year+1900, parsed.tm_mon+1, parsed.tm_mday, 
-       parsed.tm_hour, parsed.tm_min, parsed.tm_sec,
-       ts.tv_nsec/1000, cyn, tz_hour, tz_min);
   sqlBuild_Set_Str (b, tag, buf);
 }
 
@@ -306,10 +286,9 @@ sqlBuild_Where_Str (sqlBuilder *b, const char *tag, const char *val)
 void
 sqlBuild_Where_GUID (sqlBuilder *b, const char *tag, const GUID *val)
 {
-  char *guid;
+  char guid_str[GUID_ENCODING_LENGTH+1];
-  guid = guid_to_string(val);
+  guid_to_string_buff(val, guid_str);
-  sqlBuild_Where_Str (b, tag, guid);
+  sqlBuild_Where_Str (b, tag, guid_str);
-  g_free (guid);
 }
 
 /* ================================================ */

src/engine/sql/design.txt

@@ -0,0 +1,55 @@
+
+SQL Tables
+----------
+These mostly parallel the data structures in the gnc engine.
+See gnc-init.txt for more info.
+
+
+Session Table, Session Modes
+----------------------------
+There are three basic modes for accessing the database:
+"Single User",  "Polled Multi-User" and "Event-Driven Multi-User"
+The session table in the database indicates which mode the database is
+functioning.
+
+-- "Single User File Mode" -- 
+   Only one user can have access to the database at a time.  The
+   database is used as a glorified file: engine data is 'saved' to
+   the database only when the user selects 'save' from the GUI
+   dialog.  This mode exists because it is easy to implement, easy 
+   to debug, and has reasonable demands on the database for small
+   datasets.
+
+   This mode is guarenteed to clobber any sort of changes made by
+   other users (in the same way that multiple file writers clobber
+   each other).  Thus, this mode is mutually exclusive of any other,
+   and te engine prevents more than one concurrent user.  
+
+
+-- "Single User Update Mode" -- 
+   Similar to the "Single User File Mode", except that updates
+   are stored in the DB as they are made (thus eliminating the 
+   need for a big 'save' at the end of a session).  This mode is 
+   more robust in that there is minimal/no data loss in the event 
+   of a crash.
+
+-- "Multi-User Polled" -- 
+   Multiple users are assumed, Gnucash polls the database to detect 
+   changes in the data.  Partially Implemented. 
+
+
+-- "Multi-User Events" --
+   Gnucash uses the SQL LISTEN/NOTIFY async message delivery routines.
+   Not implemented.
+
+
+Session Design Notes
+--------------------
+The pgendSyncSingleFile() subroutine performs the equivalent of 'file
+save'.  Note that it does this by deleting the entire contents of the 
+database, and then writing out the entire contents of the engine.  It
+works this way (needs to work this way) in order to make sure that
+deleted transactions,etc. are really deleted from the database.  This
+is because in this mode, the backend never finds out about deletions.
+If you want incremental deletion, then use the 'Single Update' mode.
+

src/engine/sql/gnc-init.sql

@@ -1,17 +1,35 @@
 
--- these tables roughly mirror the c structs in 
+-- FILE:
--- TransactionP.h,  AccountP.h
+-- gnc-init.sql
--- these tables are hand-built, but maybe they should be 
+--
--- autobuilt with the m4 macros ...
+-- FUNCTION:
+-- Define the tables needed to initialize a new GnuCash database
+--
+-- These tables roughly mirror the c structs in 
+-- TransactionP.h,  AccountP.h, gnc-commodity.c
+-- Please refer to the C files to get the right level of documentation.
+--
+-- These tables are specifically designed for the 
+-- postgres database server, but are hopefull relatively portable.
+--
+-- These tables are hand-built, but maybe they should be 
+-- auto-built with the m4 macros ...
+--
+-- HISTORY:
+-- Copyright (C) 2000, 2001 Linas Vepstas
+--
 
 -- Commodity structure
+-- Store currency, security types.  Namespace includes
+-- ISO4217 for currencies, NASDAQ, AMEX, NYSE, EUREX for 
+-- stocks.   See the C documentation for details.
 
 DROP TABLE gncCommodity;
 CREATE TABLE gncCommodity (
         commodity	TEXT PRIMARY KEY,
 	fullname	TEXT,
-	namespace	TEXT,
+	namespace	TEXT NOT NULL,
-	mnemonic	TEXT,
+	mnemonic	TEXT NOT NULL,
 	code		TEXT,
 	fraction	INT DEFAULT '100'
 );
@@ -20,20 +38,18 @@ CREATE TABLE gncCommodity (
 -- guid. There is no supports for Groups in this schema.
 -- (there seems to be no strong need to have groups in the DB.)
 --
--- hack alert -- add the kvp frames, the currency tables,
+-- hack alert -- add kvp frames, 
-              -- the current balances, etc
 
 DROP TABLE gncAccount;
 CREATE TABLE gncAccount (
 	accountGuid	CHAR(32) PRIMARY KEY,
-	parentGuid	CHAR(32),
+	parentGuid	CHAR(32) NOT NULL,
---	childrenGuid	CHAR(32),
+	accountName 	TEXT NOT NULL CHECK (accountName <> ''),
-	accountName 	TEXT DEFAULT 'xoxo',
 	accountCode 	TEXT,
 	description 	TEXT,
 	notes	 	TEXT,
-	type		TEXT,
+	type		TEXT NOT NULL,
-	commodity	TEXT
+	commodity	TEXT NOT NULL CHECK (commodity <>'')
 );
 
 -- CREATE INDEX gncAccount_pg_idx ON gncAccount (parentGuid);
@@ -48,54 +64,71 @@ CREATE TABLE gncTransaction (
 	date_posted	 	DATETIME,
 	num			TEXT,
 	description		TEXT,
-        currency                TEXT
+        currency                TEXT NOT NULL CHECK (currency <> '')
 );
 
+CREATE INDEX gncTransaction_posted_idx ON gncTransaction (date_posted);
+
 -- a gncEntry is what we call 'Split' elsewhere in the engine
+-- Here, we call it a 'journal entry'
 
 DROP TABLE gncEntry;
 CREATE TABLE gncEntry (
 	entryGuid		CHAR(32) PRIMARY KEY,
-	accountGuid		CHAR(32),
+	accountGuid		CHAR(32) NOT NULL,
-	transGuid		CHAR(32),
+	transGuid		CHAR(32) NOT NULL,
 	memo			TEXT,
 	action			TEXT,
 	reconciled		CHAR DEFAULT 'n',
 	date_reconciled 	DATETIME,
 	amountNum		INT8 DEFAULT '0',
-	amountDenom		INT8 DEFAULT '100',
+	amountDenom		INT4 DEFAULT '100',
 	valueNum		INT8 DEFAULT '0',
-	valueDenom		INT8 DEFAULT '100'
+	valueDenom		INT4 DEFAULT '100'
 );
 
 CREATE INDEX gncEntry_acc_idx ON gncEntry (accountGuid);
 CREATE INDEX gncEntry_trn_idx ON gncEntry (transGuid);
 
--- populate with some bogus data
+-- The checkpoint table provides balance information
--- INSERT INTO gncAccount (accountGuid, parentGuid, accountName, type, description) VALUES
+-- The balance is provided in the indicated currency; 
---     ('9101752f77d6615dcdc0fffe24f0de24',
+-- this allows the potential of maintaining balance information
---      '00000000000000000000000000000000',
+-- in multiple currencies.  
---      'Swipe Trading Account', 
+-- (e.g. report stock account balances in shares of stock, 
---      'STOCK',
+-- and in dollars)
---      'Swipe Brokers Margin Account');
+
--- INSERT INTO gncAccount (accountGuid, parentGuid, accountName, type, description) VALUES
+DROP TABLE gncCheckpoint;
---     ('0d7c1819693c85c16d5556b37f6caf9d',
+CREATE TABLE gncCheckpoint (
---      '00000000000000000000000000000000',
+	accountGuid		CHAR(32) NOT NULL,
---      'Stock Dividends &amp; Distributions',
+	date_xpoint	 	DATETIME NOT NULL,
---      'BANK',
+	commodity		TEXT NOT NULL CHECK (commodity <>''),
---      'Stock Dividends &amp; Distributions');
+	balance			INT8 DEFAULT '0',
--- 
+	cleared_balance		INT8 DEFAULT '0',
--- INSERT INTO gncTransaction (transGuid, date_entered, 
+	reconciled_balance	INT8 DEFAULT '0',
---                 date_posted, num, description) VALUES
+
---     ('2ebc806e72c17bdc3c2c4e964b82eff8',
+        PRIMARY KEY (accountGuid, date_xpoint, commodity)
---      '1998-07-01 11:00:00.345678 -0500',
+);
---      '1998-07-02 11:00:00.678945 -0531',
+
---      '101aaa',
+
---      'Interest at 3.5%');
+-- The session directory serves several purposes.  First and formost,
--- 
+-- it notes the database access type.  There are three modes:
--- INSERT INTO gncEntry (entryGuid, memo, reconciled, amountNum,valueNum) VALUES
+--  o "Single User" -- Only one user can have access to the database
---     ('d56a1146e414a30d6f2e251af2075f71',
+--                     at a time. 
---      'this is a split memo',
+--  o "Multi-User Polled" -- multiple users
---      'Y',
+--  o "Muilti-User Event Driven" 
---      700000,
+--  See Design.txt for more info.
---      700000);
+-- Note that a client can lie about its identity, sign-on time, etc.
+-- so these records aren't really sufficient for a true audit.
+
+DROP TABLE gncSession;
+CREATE TABLE gncSession (
+	sessionGuid		CHAR(32) PRIMARY KEY,
+	session_mode		CHAR(16) NOT NULL,
+	hostname		TEXT,
+	login_name		TEXT,
+	gecos			TEXT,
+	time_on			DATETIME NOT NULL,
+	time_off		DATETIME NOT NULL DEFAULT 'INFINITY'
+);
+
+

src/engine/sql/gncquery.c

@@ -108,7 +108,6 @@ sqlQuery_build (sqlQuery*sq, Query *q)
             {
                int got_more = 0;
                GList *acct;
-               char guid_str[GUID_ENCODING_LENGTH+1];
 
                PINFO("term is PR_ACCOUNT");
 
@@ -119,9 +118,8 @@ sqlQuery_build (sqlQuery*sq, Query *q)
                   if (got_more) sq->pq = stpcpy(sq->pq, " AND ");
                   got_more = 1;
 
-                  guid_to_string_buff ((GUID*) acct->data, guid_str);
                   sq->pq = stpcpy(sq->pq, "accountguid='");
-                  sq->pq = stpcpy(sq->pq, guid_str);
+                  sq->pq = guid_to_string_buff ((GUID*) acct->data, sq->pq);
                   sq->pq = stpcpy(sq->pq, "'");
                }
                break;

src/engine/sql/table.m4

@@ -5,7 +5,7 @@ changecom(`/*', `*/')
 /* data dictionary for the gnucash tables */
 /* sql table description and manipulation macros */
 
-define(`account', `gncAccount, Account,
+define(`account', `gncAccount, Account, Account,
        accountName,    , char *, xaccAccountGetName(ptr),
        accountCode,    , char *, xaccAccountGetCode(ptr),
        description,    , char *, xaccAccountGetDescription(ptr),
@@ -17,7 +17,7 @@ define(`account', `gncAccount, Account,
        ')
 
 
-define(`split', `gncEntry, Split,
+define(`split', `gncEntry, Split, Split,
        accountGUID,     , GUID *,   xaccAccountGetGUID(xaccSplitGetAccount(ptr)),
        transGUID,       , GUID *,   xaccTransGetGUID(xaccSplitGetParent(ptr)),
        memo,            , char *,   xaccSplitGetMemo(ptr),
@@ -38,7 +38,7 @@ define(`split', `gncEntry, Split,
 /* date_entered,    , Timespec, xaccTransRetDateEnteredTS(ptr), */
 /* as one might have guessed                                    */
 
-define(`transaction', `gncTransaction, Transaction,
+define(`transaction', `gncTransaction, Transaction, Transaction,
        num,            , char *,   xaccTransGetNum(ptr),
        description,    , char *,   xaccTransGetDescription(ptr),
        currency,       , char *,   gnc_commodity_get_unique_name(xaccTransGetCurrency(ptr)),
@@ -48,7 +48,7 @@ define(`transaction', `gncTransaction, Transaction,
        ')
 
 
-define(`modity', `gncCommodity, gnc_commodity,
+define(`modity', `gncCommodity, Commodity, gnc_commodity, 
        namespace,    , char *, gnc_commodity_get_namespace(ptr),
        fullname,     , char *, gnc_commodity_get_fullname(ptr),
        mnemonic,     , char *, gnc_commodity_get_mnemonic(ptr),
@@ -58,12 +58,33 @@ define(`modity', `gncCommodity, gnc_commodity,
        ')
        
 
+define(`checkpoint', `gncCheckpoint, Checkpoint, Checkpoint,
+       balance,             , int64,    ptr->balance,
+       cleared_balance,     , int64,    ptr->cleared_balance,
+       reconciled_balance,  , int64,    ptr->reconciled_balance,
+       date_xpoint,         , Timespec, ptr->datetime,
+       commodity,           , char *,   ptr->commodity,
+       accountGuid,         , GUID *,   ptr->account_guid,
+       ')
+       
+
+define(`session', `gncSession, Session, void,
+       session_mode,        , char *, pgendSessionGetMode(be),
+       hostname,            , char *, pgendGetHostname(be),
+       login_name,          , char *, pgendGetUsername(be),
+       gecos,               , char *, pgendGetUserGecos(be),
+       time_on,             , now,    "NOW",
+       time_off,            , now,    "INFINITY",
+       sessionGUID,      KEY, GUID *, be->sessionGuid,
+       ')
+       
 /* ------------------------------------------------------- */
 /* symbolic names for the table accessors */
 define(`tablename', $1)
-define(`xacc_type', $2)
+define(`func_name', $2)
+define(`xacc_type', $3)
 
-define(`firstrec', `shift(shift($@))')
+define(`firstrec', `shift(shift(shift($@)))')
 define(`nextrec', `shift(shift(shift(shift($@))))')
 
 /* -------- */
@@ -123,7 +144,7 @@ define(`store_one_only',
  */
 
 static void 
-pgendStoreOne`'xacc_type($@)`'Only (PGBackend *be,
+pgendStoreOne`'func_name($@)`'Only (PGBackend *be,
                      xacc_type($@) *ptr,
                      sqlBuild_QType update)
 {
@@ -155,7 +176,7 @@ define(`compare_one_only',
  */
 
 static int
-pgendCompareOne`'xacc_type($@)`'Only (PGBackend *be,
+pgendCompareOne`'func_name($@)`'Only (PGBackend *be,
                      xacc_type($@) *ptr)
 {
    const char *buf;
@@ -192,14 +213,44 @@ pgendCompareOne`'xacc_type($@)`'Only (PGBackend *be,
 
 ')
 
+define(`put_one_only', 
+`
+/* ------------------------------------------------------ */
+/* This routine inserts or updates, as appropriate
+ * It does not do any traversals, it does not lock.  
+ * It just updates.
+ */
+
+static void 
+pgendPutOne`'func_name($@)`'Only (PGBackend *be,
+                     xacc_type($@) *ptr)
+{
+   int ndiffs;
+   ndiffs = pgendCompareOne`'func_name($@)`'Only (be, ptr);
+
+   /* update the record if there are differences ... */
+   if (0<ndiffs) pgendStoreOne`'func_name($@)`'Only (be, ptr, SQL_UPDATE);
+   /* insert the record if it doesnt exist */
+   if (0>ndiffs) pgendStoreOne`'func_name($@)`'Only (be, ptr, SQL_INSERT);
+}
+
+')
+
 /* ------------------------------------------------------- */
 divert
 store_one_only(account)
-store_one_only(transaction)
+store_one_only(checkpoint)
-store_one_only(split)
 store_one_only(modity)
+store_one_only(session)
+store_one_only(split)
+store_one_only(transaction)
 
 compare_one_only(account)
-compare_one_only(transaction)
-compare_one_only(split)
 compare_one_only(modity)
+compare_one_only(split)
+compare_one_only(transaction)
+
+put_one_only(account)
+put_one_only(modity)
+put_one_only(split)
+put_one_only(transaction)