Bill Gribble's qif importing patches.

git-svn-id: svn+ssh://svn.gnucash.org/repo/gnucash/trunk@3674 57a11ea4-9604-0410-9ed3-97b8803252fd

ChangeLog

@@ -1,3 +1,11 @@
+2001-02-15  Bill Gribble  <grib@billgribble.com>
+
+	* src/scm/qif-import/qif-guess-map.scm: add stock hash
+	save/restore to map IO
+
+	* src/scm/qif-import/qif-to-gnc.scm: bug fixes to remove duplicate
+	account problems
+	
 2001-02-15  Christian Stimming  <stimming@tuhh.de>
 
 	* src/scm/report-utilities.scm: Added functions to filter accounts

src/engine/gnc-commodity.c

@@ -499,7 +499,7 @@ gnc_commodity_table_add_namespace(gnc_commodity_table * table,
  ********************************************************************/
 
 static int
-ns_helper(gpointer key, gpointer value, gpointer user_data) {
+ns_helper(gpointer key, gpoin4er value, gpointer user_data) {
   gnc_commodity * c = value;
   gnc_commodity_destroy(c);
   g_free(key);

src/gnome/druid-qif-import.c

@@ -86,6 +86,7 @@ struct _qifimportwindow {
 
   SCM       gnc_acct_info;
   SCM       stock_hash;
+  SCM       new_stocks;
 
   SCM       imported_account_group;
   SCM       match_transactions;
@@ -167,6 +168,7 @@ gnc_ui_qif_import_druid_make(void)  {
   retval->memo_display_info =  SCM_BOOL_F;
   retval->memo_map_info     =  SCM_BOOL_F;
   retval->stock_hash        =  SCM_BOOL_F;
+  retval->new_stocks        =  SCM_BOOL_F;
   retval->imported_account_group   = SCM_BOOL_F;
   retval->match_transactions = SCM_BOOL_F;
   retval->selected_transaction = 0;
@@ -223,6 +225,7 @@ gnc_ui_qif_import_druid_make(void)  {
   retval->acct_map_info    = gh_list_ref(mapping_info, gh_int2scm(1));
   retval->cat_map_info     = gh_list_ref(mapping_info, gh_int2scm(2));
   retval->memo_map_info    = gh_list_ref(mapping_info, gh_int2scm(3));
+  retval->stock_hash       = gh_list_ref(mapping_info, gh_int2scm(4));
   
   scm_protect_object(retval->imported_files);
   scm_protect_object(retval->selected_file);
@@ -234,6 +237,7 @@ gnc_ui_qif_import_druid_make(void)  {
   scm_protect_object(retval->acct_display_info);
   scm_protect_object(retval->acct_map_info);
   scm_protect_object(retval->stock_hash);
+  scm_protect_object(retval->new_stocks);
   scm_protect_object(retval->imported_account_group);
   scm_protect_object(retval->match_transactions);
   
@@ -278,6 +282,7 @@ gnc_ui_qif_import_druid_destroy (QIFImportWindow * window) {
   scm_unprotect_object(window->acct_display_info);
   scm_unprotect_object(window->acct_map_info);
   scm_unprotect_object(window->stock_hash);
+  scm_unprotect_object(window->new_stocks);
   scm_unprotect_object(window->imported_account_group);
   scm_unprotect_object(window->match_transactions);
 
@@ -1283,7 +1288,6 @@ gnc_ui_qif_import_convert(QIFImportWindow * wind) {
   /* get the default currency */
   char * currname = gtk_entry_get_text(GTK_ENTRY(wind->currency_entry));
 
-
   /* busy cursor */
   gnc_suspend_gui_refresh ();
   gnc_set_busy_cursor(NULL);
@@ -1402,8 +1406,9 @@ gnc_ui_qif_import_memo_next_cb(GnomeDruidPage * page,
   QIFImportWindow * wind = 
     gtk_object_get_data(GTK_OBJECT(user_data), "qif_window_struct");
   
-  SCM any_new   = gh_eval_str("qif-import:any-new-accts?");
+  SCM any_new      = gh_eval_str("qif-import:any-new-accts?");
-  SCM any_stock = gh_eval_str("qif-import:any-new-stock-accts?");
+  SCM update_stock = gh_eval_str("qif-import:update-stock-hash");
+
   int show_matches;
   
   /* if any accounts are new, ask about the currency; else,
@@ -1416,7 +1421,12 @@ gnc_ui_qif_import_memo_next_cb(GnomeDruidPage * page,
   else {
     /* if we need to look at stocks, do that, otherwise import
      * xtns and go to the duplicates page */
-    if(gh_call1(any_stock, wind->acct_map_info) == SCM_BOOL_T) {
+    scm_unprotect_object(wind->new_stocks);
+    wind->new_stocks = gh_call2(update_stock, wind->stock_hash, 
+                                wind->acct_map_info);
+    scm_protect_object(wind->new_stocks);
+    
+    if(wind->new_stocks != SCM_BOOL_F) {
       if(wind->show_doc_pages) {
         gnome_druid_set_page(GNOME_DRUID(wind->druid),
                              get_named_page(wind, "commodity_doc_page"));
@@ -1463,10 +1473,15 @@ gnc_ui_qif_import_currency_next_cb(GnomeDruidPage * page,
                                    gpointer user_data) {
   QIFImportWindow * wind = 
     gtk_object_get_data(GTK_OBJECT(user_data), "qif_window_struct");
-  SCM any_stock = gh_eval_str("qif-import:any-new-stock-accts?");
+  SCM update_stock = gh_eval_str("qif-import:update-stock-hash");
   int show_matches;
 
-  if(gh_call1(any_stock, wind->acct_map_info) == SCM_BOOL_T) {
+  scm_unprotect_object(wind->new_stocks);
+  wind->new_stocks =  gh_call2(update_stock, wind->stock_hash, 
+                               wind->acct_map_info);
+  scm_protect_object(wind->new_stocks);
+  
+  if(wind->new_stocks != SCM_BOOL_F) {
     if(wind->show_doc_pages) {
       gnome_druid_set_page(GNOME_DRUID(wind->druid),
                            get_named_page(wind, "commodity_doc_page"));
@@ -1569,10 +1584,9 @@ gnc_ui_qif_import_commodity_prepare_cb(GnomeDruidPage * page,
     gtk_object_get_data(GTK_OBJECT(user_data), "qif_window_struct");
 
   SCM   hash_ref          = gh_eval_str("hash-ref");
-  SCM   setup_stock_hash  = gh_eval_str("qif-import:setup-stock-hash");
+  SCM   stocks;
-  SCM   setup_info;
-  SCM   stock_names;
   SCM   comm_ptr_token;
+  SCM   show_matches;
 
   gnc_commodity  * commodity;
   GnomeDruidPage * back_page = get_named_page(wind, "commodity_doc_page");  
@@ -1581,17 +1595,32 @@ gnc_ui_qif_import_commodity_prepare_cb(GnomeDruidPage * page,
   /* only set up once */
   if(wind->commodity_pages) return;
   
-  /* make a list of the new stocks that we need info about */
+  /* this shouldn't happen, but DTRT if it does */
-  setup_info       = gh_call1(setup_stock_hash, wind->acct_map_info);  
+  if(gh_null_p(wind->new_stocks)) {
-  stock_names      = gh_cadr(setup_info);
+    printf("somehow got to commodity doc page with nothing to do... BUG!\n");
+    show_matches = gnc_ui_qif_import_convert(wind);
     
-  scm_unprotect_object(wind->stock_hash);
+    if(show_matches) {
-  wind->stock_hash = gh_car(setup_info);
+      if(wind->show_doc_pages) {
-  scm_protect_object(wind->stock_hash);
+        /* check for matches .. the docpage does it automatically */ 
+        gnome_druid_set_page(GNOME_DRUID(wind->druid),
+                             get_named_page(wind, "match_doc_page"));
+      }
+      else {
+        gnome_druid_set_page(GNOME_DRUID(wind->druid),
+                             get_named_page(wind, "match_duplicates_page"));
+      }
+    }
+    else {
+      gnome_druid_set_page(GNOME_DRUID(wind->druid),
+                           get_named_page(wind, "end_page"));
+    } 
+  }
 
   /* insert new pages, one for each stock */
-  while(!gh_null_p(stock_names)) {
+  stocks = wind->new_stocks;
-    comm_ptr_token = gh_call2(hash_ref, wind->stock_hash, gh_car(stock_names));
+  while(!gh_null_p(stocks)) {
+    comm_ptr_token = gh_call2(hash_ref, wind->stock_hash, gh_car(stocks));
     commodity      = gw_wcp_get_ptr(comm_ptr_token);
     
     new_page = make_qif_druid_page(commodity);
@@ -1608,7 +1637,7 @@ gnc_ui_qif_import_commodity_prepare_cb(GnomeDruidPage * page,
                             GNOME_DRUID_PAGE(new_page->page));
     back_page = GNOME_DRUID_PAGE(new_page->page);
     
-    stock_names = gh_cdr(stock_names);
+    stocks = gh_cdr(stocks);
     gtk_widget_show_all(new_page->page);
   }
 }
@@ -1833,8 +1862,9 @@ gnc_ui_qif_import_finish_cb(GnomeDruidPage * gpage,
   gnc_resume_gui_refresh();
   
   /* write out mapping info before destroying the window */
-  gh_call3(save_map_prefs, wind->acct_map_info, wind->cat_map_info,
+  gh_apply(save_map_prefs, 
-           wind->memo_map_info); 
+           SCM_LIST4(wind->acct_map_info, wind->cat_map_info,
+                     wind->memo_map_info, wind->stock_hash));
   
   gnc_ui_qif_import_druid_destroy(wind);  
 }

src/scm/qif-import/qif-dialog-utils.scm

@@ -544,25 +544,6 @@
      (vector->list hash-table))
     retval))
 
-(define (qif-import:any-new-stock-accts? hash-table) 
-  (let ((retval #f))
-    (for-each 
-     (lambda (bin)
-       (for-each 
-        (lambda (elt)
-          (if (and 
-               (qif-map-entry:new-acct? (cdr elt))
-               (qif-map-entry:display? (cdr elt))
-               (or 
-                (memv GNC-STOCK-TYPE 
-                      (qif-map-entry:allowed-types (cdr elt)))
-                (memv GNC-MUTUAL-TYPE 
-                      (qif-map-entry:allowed-types (cdr elt)))))
-              (set! retval #t)))
-        bin))
-     (vector->list hash-table))
-    retval))
-
 (define (qif-import:fix-from-acct qif-file new-acct-name) 
   (for-each 
    (lambda (xtn)
@@ -589,42 +570,59 @@
                 (match:substring match 2))))
         fullname)))
 
-(define (qif-import:setup-stock-hash hash-table)
-  (let ((newhash (make-hash-table 20))
-        (names '()))
-    (for-each 
-     (lambda (bin)
-       (for-each 
-        (lambda (elt)
-          (if (and 
-               (qif-map-entry:new-acct? (cdr elt))
-               (qif-map-entry:display? (cdr elt)) 
-               (or 
-                (memv GNC-STOCK-TYPE 
-                      (qif-map-entry:allowed-types (cdr elt)))
-                (memv GNC-MUTUAL-TYPE 
-                      (qif-map-entry:allowed-types (cdr elt)))))
-              (let* ((name (qif-map-entry:qif-name (cdr elt)))
-                     (stock-name (qif-import:get-account-name name)))
-                (if (not stock-name)
-                    (begin
-                      (display "stock-name #f.. name ==")
-                      (display name)(newline)))
 
-                (if (not (hash-ref newhash stock-name))
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-                    (begin 
+;;  qif-import:update-stock-hash
-                      (set! names (cons stock-name names))
+;;
-                      (hash-set! newhash stock-name 
+;;  make new commodities for each new stock in acct-hash that isn't
-                                 (gnc:commodity-create 
+;;  already in stock-hash.  Return a list of the QIF names of the
-                                  stock-name
+;;  new stocks or #f if none.
-                                  GNC_COMMODITY_NS_NYSE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-                                  stock-name
-                                  ""
-                                  100000)))))))
-        bin))
-     (vector->list hash-table))
-    (list newhash (sort names string<?))))
 
+(define (qif-import:update-stock-hash stock-hash acct-hash)
+  (let ((names '()))
+    (hash-fold 
+     (lambda (qif-name map-entry p)
+       (let ((stock-name (qif-import:get-account-name qif-name)))         
+         ;; is it: a stock or mutual fund and displayed and not already in
+         ;; the stock-hash?
+         (if (and 
+              stock-name 
+              (qif-map-entry:display? map-entry)
+              (or (memv GNC-STOCK-TYPE 
+                        (qif-map-entry:allowed-types map-entry))
+                  (memv GNC-MUTUAL-TYPE 
+                        (qif-map-entry:allowed-types map-entry)))
+              (not (hash-ref stock-hash stock-name)))
+             (let* ((separator (string-ref (gnc:account-separator-char) 0))
+                    (existing-gnc-acct 
+                     (gnc:get-account-from-full-name 
+                      (gnc:get-current-group)
+                      (qif-map-entry:gnc-name map-entry)
+                      separator)))
+               (if existing-gnc-acct
+                   ;; gnc account already exists... we *know* what the 
+                   ;; security is supposed to be 
+                   (hash-set! 
+                    stock-hash stock-name
+                    (gnc:account-get-security existing-gnc-acct))
+                   ;; we know nothing about this security.. we need to 
+                   ;; ask about it
+                   (begin 
+                     (set! names (cons stock-name names))
+                     (hash-set! 
+                      stock-hash stock-name 
+                      (gnc:commodity-create stock-name
+                                            GNC_COMMODITY_NS_NYSE
+                                            stock-name
+                                            ""
+                                            100000))))))
+         #f))
+     #f acct-hash)
+
+    (if (not (null? names))
+        (sort names string<?)
+        #f)))
 
 ;; this is used within the dialog to get a list of all the new
 ;; accounts the importer thinks it's going to make.  Passed to the

src/scm/qif-import/qif-guess-map.scm

@@ -57,13 +57,15 @@
     (false-if-exception 
      (read)))
 
-  ;; we'll be returning a list of 4 elements:  
+  ;; we'll be returning a list:  
   ;;  - a list of all the known gnucash accounts in 
-  ;;    (shortname fullname account) format.
+  ;;    (shortname fullname account*) format.
   ;;  - a hash of QIF account name to gnucash account info
   ;;  - a hash of QIF category to gnucash account info
   ;;  - a hash of QIF memo/payee to gnucash account info  
   ;;    (older saved prefs may not have this one)
+  ;;  - a hash of QIF stock name to gnc-commodity*
+  ;;    (older saved prefs may not have this one)
   (let* ((pref-dir (build-path (getenv "HOME") ".gnucash"))
          (pref-filename (build-path pref-dir "qif-accounts-map"))
          (results '()))
@@ -78,9 +80,11 @@
             (let ((qif-account-list #f)
                   (qif-cat-list #f)
                   (qif-memo-list #f)
+                  (qif-stock-list #f)
                   (qif-account-hash #f)
                   (qif-cat-hash #f)
-                  (qif-memo-hash #f))
+                  (qif-memo-hash #f)
+                  (qif-stock-hash #f))
               (set! qif-account-list (safe-read))
               (if (not (list? qif-account-list))
                   (set! qif-account-hash (make-hash-table 20))
@@ -97,10 +101,17 @@
                   (set! qif-memo-hash (make-hash-table 20))
                   (set! qif-memo-hash (qif-import:read-map qif-memo-list)))
 
+              (set! qif-stock-list (safe-read))
+              (if (not (list? qif-stock-list))
+                  (set! qif-stock-hash (make-hash-table 20))
+                  (set! qif-stock-hash (qif-import:read-commodities
+                                        qif-stock-list)))              
               (set! results 
-                    (list qif-account-hash qif-cat-hash qif-memo-hash)))))
+                    (list qif-account-hash qif-cat-hash 
+                          qif-memo-hash qif-stock-hash)))))
         (begin 
           (set! results (list (make-hash-table 20)
+                              (make-hash-table 20)
                               (make-hash-table 20)
                               (make-hash-table 20)))))
     
@@ -121,15 +132,10 @@
 
 (define (qif-import:write-map hashtab)
   (let ((table '()))
-    (for-each 
+    (hash-fold 
-     (lambda (bin)
+     (lambda (key value p)
-       (for-each 
+       (set! table (cons (cons key (simple-obj-to-list value)) table))
-        (lambda (entry)
+       #f) #f hashtab)
-          (let ((key (car entry))
-                (value (cdr entry)))
-            (set! table (cons (cons key (simple-obj-to-list value)) table))))
-        bin))
-     (vector->list hashtab))
     (write table)))
 
 (define (qif-import:read-map tablist)
@@ -143,7 +149,36 @@
      tablist)
     table))
 
-(define (qif-import:save-map-prefs acct-map cat-map memo-map)
+(define (qif-import:read-commodities commlist)
+  (let ((table (make-hash-table 20)))
+    (for-each 
+     (lambda (entry)
+       (if (and (list? entry) 
+                (= 3 (length entry)))
+           (let ((name (car entry))
+                 (namespace (cadr entry))
+                 (mnemonic (caddr entry)))
+             (hash-set! table name
+                        (gnc:commodity-table-lookup (gnc:engine-commodities) 
+                                                    namespace mnemonic)))))
+     commlist)
+    table))
+                              
+(define (qif-import:write-commodities hashtab)
+  (let ((table '()))
+    (hash-fold
+     (lambda (key value p)
+       (if (gw:wcp-is-of-type? <gnc:commodity*> value)
+           (set! table
+                 (cons (list key 
+                             (gnc:commodity-get-namespace value)
+                             (gnc:commodity-get-mnemonic value))
+                       table)))
+       #f) #f hashtab)
+    (write table)))
+        
+
+(define (qif-import:save-map-prefs acct-map cat-map memo-map stock-map)
   (let* ((pref-dir (build-path (getenv "HOME") ".gnucash"))
          (pref-filename (build-path pref-dir "qif-accounts-map"))
          (save-ok #f))
@@ -168,12 +203,18 @@
             
             (display ";;; map from QIF accounts to GNC accounts") (newline) 
             (qif-import:write-map acct-map)
+            (newline)
 
             (display ";;; map from QIF categories to GNC accounts") (newline)
             (qif-import:write-map cat-map)
+            (newline)
 
             (display ";;; map from QIF payee/memo to GNC accounts") (newline)
             (qif-import:write-map memo-map)
+            (newline)
+
+            (display ";;; map from QIF stock name to GNC commodity") (newline)
+            (qif-import:write-commodities stock-map)           
             (newline))))))
 
 

src/scm/qif-import/qif-parse.scm

@@ -154,12 +154,11 @@
      ((string=? mangled-string "invst") ;; these are brokerage accounts.
       (list GNC-BANK-TYPE))
      ((string=? mangled-string "oth a")
-      (list GNC-ASSET-TYPE GNC-BANK-TYPE GNC-CASH-TYPE GNC-STOCK-TYPE 
+      (list GNC-ASSET-TYPE GNC-BANK-TYPE GNC-CASH-TYPE))
-            GNC-MUTUAL-TYPE))
      ((string=? mangled-string "oth l")
       (list GNC-LIABILITY-TYPE GNC-CCARD-TYPE))
      ((string=? mangled-string "mutual")
-      (list GNC-MUTUAL-TYPE GNC-STOCK-TYPE))
+      (list GNC-BANK-TYPE))
      (#t
       (display "qif-parse:parse-acct-type : unhandled account type ")
       (display read-value)

src/scm/qif-import/qif-to-gnc.scm

@@ -41,19 +41,28 @@
                   (string-append short-name (sprintf #f " %a" count)))))
           short-name))
     
+    (define (compatible? account)
+      (let ((acc-type (gnc:account-get-type account))
+            (acc-currency (gnc:account-get-currency account))
+            (acc-security (gnc:account-get-security account)))
+        (if (memv acc-type 
+                  (list GNC-STOCK-TYPE GNC-MUTUAL-TYPE))
+            (and 
+             (list? allowed-types)
+             (memv acc-type allowed-types)
+             (gnc:commodity-equiv? acc-currency currency)
+             (gnc:commodity-equiv? acc-security security))
+            (and 
+             (list? allowed-types)
+             (memv acc-type allowed-types)
+             (gnc:commodity-equiv? acc-currency currency)))))
+                 
     ;; just because we found an account doesn't mean we can use it.
     ;; if the name is in use but the currency, security, or type are
     ;; incompatible, we need to create a new account with a modified
     ;; name.
     (if same-gnc-account 
-        (if (and (gnc:commodity-equiv? 
+        (if (compatible? same-gnc-account)
-                  currency (gnc:account-get-currency same-gnc-account))
-                 (or (eq? (gnc:account-get-security same-gnc-account) #f)
-                     (gnc:commodity-equiv? 
-                      security (gnc:account-get-security same-gnc-account)))
-                 (list? allowed-types)
-                 (memv (gnc:account-get-type same-gnc-account)
-                       allowed-types))
             (begin 
               ;; everything is ok, so we can just use the same
               ;; account.  Make sure we make the same type.