diff --git a/gnucash/import-export/CMakeLists.txt b/gnucash/import-export/CMakeLists.txt index 9882a0234a..11591f3d95 100644 --- a/gnucash/import-export/CMakeLists.txt +++ b/gnucash/import-export/CMakeLists.txt @@ -16,7 +16,7 @@ add_subdirectory(qif-imp) set (generic_import_SOURCES import-account-matcher.c import-commodity-matcher.c - import-backend.c + import-backend.cpp import-format-dialog.c import-match-picker.c import-parse.c diff --git a/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp b/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp index 34f8d4dd88..f70c51389b 100644 --- a/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp +++ b/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp @@ -50,6 +50,7 @@ #include "import-account-matcher.h" #include "import-main-matcher.h" +#include "import-backend.h" #include "gnc-csv-account-map.h" #include "gnc-account-sel.h" @@ -1732,8 +1733,6 @@ void CsvImpTransAssist::preview_validate_settings () /* Populates the account match view with all potential * account names found in the parse data. - * - * @param info The data being previewed */ void CsvImpTransAssist::acct_match_set_accounts () { @@ -1963,7 +1962,7 @@ CsvImpTransAssist::assist_account_match_page_prepare () // Load the account strings into the store acct_match_set_accounts (); - // Match the account strings to the mappings + // Match the account strings to account maps from previous imports auto store = gtk_tree_view_get_model (GTK_TREE_VIEW(account_match_view)); gnc_csv_account_map_load_mappings (store); @@ -2098,7 +2097,15 @@ CsvImpTransAssist::assist_match_page_prepare () auto draft_trans = trans_it.second; if (draft_trans->trans) { - gnc_gen_trans_list_add_trans (gnc_csv_importer_gui, draft_trans->trans); + auto lsplit = GNCImportLastSplitInfo { + draft_trans->m_price ? static_cast(*draft_trans->m_price) : gnc_numeric{0, 0}, + draft_trans->m_taction ? draft_trans->m_taction->c_str() : nullptr, + draft_trans->m_tmemo ? draft_trans->m_tmemo->c_str() : nullptr, + draft_trans->m_trec_state ? *draft_trans->m_trec_state : '\0', + draft_trans->m_trec_date ? static_cast(GncDateTime(*draft_trans->m_trec_date, DayPart::neutral)) : 0, + }; + + gnc_gen_trans_list_add_trans_with_split_data (gnc_csv_importer_gui, std::move (draft_trans->trans), &lsplit); draft_trans->trans = nullptr; } } diff --git a/gnucash/import-export/csv-imp/gnc-csv-account-map.c b/gnucash/import-export/csv-imp/gnc-csv-account-map.c index a3e9ca09af..aeb942b6e6 100644 --- a/gnucash/import-export/csv-imp/gnc-csv-account-map.c +++ b/gnucash/import-export/csv-imp/gnc-csv-account-map.c @@ -44,20 +44,6 @@ /* This static indicates the debugging module that this .o belongs to. */ static QofLogModule UNUSED_VAR log_module = G_LOG_DOMAIN; -/************************************************** - * account_imap_destroy - * - * Destroy an import map. But all stored entries will - * still continue to exist in the underlying kvp frame - * of the account. - **************************************************/ -static void -account_imap_destroy (GncImportMatchMap *imap) -{ - if (!imap) return; - g_free (imap); -} - /************************************************** * gnc_csv_account_map_search * @@ -76,17 +62,13 @@ Account * gnc_csv_account_map_search (const gchar *map_string) /* Go through list of accounts */ for (ptr = accts; ptr; ptr = g_list_next (ptr)) { - GncImportMatchMap *tmp_imap; + Account *tmp_acc = ptr->data; - tmp_imap = gnc_account_imap_create_imap (ptr->data); - - if (gnc_account_imap_find_account (tmp_imap, CSV_CATEGORY, map_string) != NULL) + if (gnc_account_imap_find_account (tmp_acc, CSV_CATEGORY, map_string)) { - account = ptr->data; - account_imap_destroy (tmp_imap); + account = tmp_acc; break; } - account_imap_destroy (tmp_imap); } g_list_free (accts); @@ -118,21 +100,19 @@ gnc_csv_account_map_load_mappings (GtkTreeModel *mappings_store) // Walk through the list, reading each row gtk_tree_model_get (GTK_TREE_MODEL(mappings_store), &iter, MAPPING_STRING, &map_string, MAPPING_ACCOUNT, &account, -1); - if (account == NULL) // if account is NULL, store has not been updated + // Look for an account matching the map_string + // It may already be set in the tree model. If not we try to match the map_string with + // - an entry in our saved account maps + // - a full name of any of our existing accounts + if (account || + (account = gnc_csv_account_map_search (map_string)) || + (account = gnc_account_lookup_by_full_name (gnc_get_current_root_account(), map_string))) { - account = gnc_csv_account_map_search (map_string); //search the account list for the map_string - - if (account == NULL) // account still NULL, we have no map - { - g_free (map_string); - valid = gtk_tree_model_iter_next (mappings_store, &iter); - continue; - } + fullpath = gnc_account_get_full_name (account); + gtk_list_store_set (GTK_LIST_STORE(mappings_store), &iter, MAPPING_FULLPATH, fullpath, -1); + gtk_list_store_set (GTK_LIST_STORE(mappings_store), &iter, MAPPING_ACCOUNT, account, -1); + g_free (fullpath); } - fullpath = gnc_account_get_full_name (account); - gtk_list_store_set (GTK_LIST_STORE(mappings_store), &iter, MAPPING_FULLPATH, fullpath, -1); - gtk_list_store_set (GTK_LIST_STORE(mappings_store), &iter, MAPPING_ACCOUNT, account, -1); - g_free (fullpath); g_free (map_string); valid = gtk_tree_model_iter_next (mappings_store, &iter); @@ -148,22 +128,12 @@ gnc_csv_account_map_load_mappings (GtkTreeModel *mappings_store) void gnc_csv_account_map_change_mappings (Account *old_account, Account *new_account, const gchar *map_string) { - GncImportMatchMap *tmp_imap; - if (strlen (map_string) == 0) return; - if (old_account != NULL) - { - tmp_imap = gnc_account_imap_create_imap (old_account); - gnc_account_imap_delete_account (tmp_imap, CSV_CATEGORY, map_string); - account_imap_destroy (tmp_imap); - } + if (old_account) + gnc_account_imap_delete_account (old_account, CSV_CATEGORY, map_string); - if (new_account != NULL) - { - tmp_imap = gnc_account_imap_create_imap (new_account); - gnc_account_imap_add_account (tmp_imap, CSV_CATEGORY, map_string, new_account); - account_imap_destroy (tmp_imap); - } + if (new_account) + gnc_account_imap_add_account (new_account, CSV_CATEGORY, map_string, new_account); } diff --git a/gnucash/import-export/csv-imp/gnc-csv-account-map.h b/gnucash/import-export/csv-imp/gnc-csv-account-map.h index 726d9e4d64..c6391e4af7 100644 --- a/gnucash/import-export/csv-imp/gnc-csv-account-map.h +++ b/gnucash/import-export/csv-imp/gnc-csv-account-map.h @@ -38,6 +38,15 @@ extern "C" { enum GncImportColumn {MAPPING_STRING, MAPPING_FULLPATH, MAPPING_ACCOUNT}; /** Load the import mappings. + * + * For each mapping string in the tree model, try to find a + * corresponding account and account full path. + * + * - if the account was already set, just update the full path + * - if the mapping string matches an account in the account maps, + * use that account and its corresponding full name + * - otherwise search for an existing account whose full name matches the + * mapping string * */ void gnc_csv_account_map_load_mappings (GtkTreeModel *mappings_store); diff --git a/gnucash/import-export/csv-imp/gnc-imp-props-tx.cpp b/gnucash/import-export/csv-imp/gnc-imp-props-tx.cpp index 8fbfc0d93c..e2894b4ae6 100644 --- a/gnucash/import-export/csv-imp/gnc-imp-props-tx.cpp +++ b/gnucash/import-export/csv-imp/gnc-imp-props-tx.cpp @@ -57,21 +57,23 @@ std::map gnc_csv_col_type_strs = { { GncTransPropType::NONE, N_("None") }, { GncTransPropType::UNIQUE_ID, N_("Transaction ID") }, { GncTransPropType::DATE, N_("Date") }, - { GncTransPropType::NUM, N_("Num") }, + { GncTransPropType::NUM, N_("Number") }, { GncTransPropType::DESCRIPTION, N_("Description") }, { GncTransPropType::NOTES, N_("Notes") }, { GncTransPropType::COMMODITY, N_("Transaction Commodity") }, { GncTransPropType::VOID_REASON, N_("Void Reason") }, { GncTransPropType::ACTION, N_("Action") }, { GncTransPropType::ACCOUNT, N_("Account") }, - { GncTransPropType::DEPOSIT, N_("Deposit") }, - { GncTransPropType::WITHDRAWAL, N_("Withdrawal") }, + { GncTransPropType::AMOUNT, N_("Amount") }, + { GncTransPropType::AMOUNT_NEG, N_("Amount (Negated)") }, { GncTransPropType::PRICE, N_("Price") }, { GncTransPropType::MEMO, N_("Memo") }, { GncTransPropType::REC_STATE, N_("Reconciled") }, { GncTransPropType::REC_DATE, N_("Reconcile Date") }, { GncTransPropType::TACTION, N_("Transfer Action") }, { GncTransPropType::TACCOUNT, N_("Transfer Account") }, + { GncTransPropType::T_AMOUNT, N_("Transfer Amount") }, + { GncTransPropType::T_AMOUNT_NEG, N_("Transfer Amount (Negated)") }, { GncTransPropType::TMEMO, N_("Transfer Memo") }, { GncTransPropType::TREC_STATE, N_("Transfer Reconciled") }, { GncTransPropType::TREC_DATE, N_("Transfer Reconcile Date") } @@ -86,6 +88,8 @@ std::vector twosplit_blacklist = { std::vector multisplit_blacklist = { GncTransPropType::TACTION, GncTransPropType::TACCOUNT, + GncTransPropType::T_AMOUNT, + GncTransPropType::T_AMOUNT_NEG, GncTransPropType::TMEMO, GncTransPropType::TREC_STATE, GncTransPropType::TREC_DATE @@ -107,13 +111,13 @@ GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split) * @return a GncNumeric * @exception May throw std::invalid argument if string can't be parsed properly */ -GncNumeric parse_amount (const std::string &str, int currency_format) +GncNumeric parse_monetary (const std::string &str, int currency_format) { /* An empty field is treated as zero */ if (str.empty()) return GncNumeric{}; - /* Strings otherwise containing not digits will be considered invalid */ + /* Strings otherwise containing no digits will be considered invalid */ if(!boost::regex_search(str, boost::regex("[0-9]"))) throw std::invalid_argument (_("Value doesn't appear to contain a valid number.")); @@ -200,31 +204,6 @@ gnc_commodity* parse_commodity (const std::string& comm_str) return comm; } - -static GncNumeric parse_price (const std::string &str) -{ - /* An empty field is treated as zero */ - if (str.empty()) - return GncNumeric{}; - - /* Strings otherwise containing not digits will be considered invalid */ - if(!boost::regex_search(str, boost::regex("[0-9]"))) - throw std::invalid_argument (_("Value doesn't appear to contain a valid number.")); - - auto expr = boost::make_u32regex("[[:Sc:]]"); - std::string str_no_symbols = boost::u32regex_replace(str, expr, ""); - - /* Convert based on user chosen currency format */ - gnc_numeric val = gnc_numeric_zero(); - char *endptr; - - auto success = gnc_exp_parser_parse (str.c_str(), &val, &endptr); - if (!success) - throw std::invalid_argument (_("Price can't be parsed into a number.")); - - return GncNumeric(val); -} - void GncPreTrans::set (GncTransPropType prop_type, const std::string& value) { try @@ -243,7 +222,12 @@ void GncPreTrans::set (GncTransPropType prop_type, const std::string& value) case GncTransPropType::DATE: m_date = boost::none; - m_date = GncDate(value, GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails + if (!value.empty()) + m_date = GncDate(value, GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails + else if (!m_multi_split) + throw std::invalid_argument ( + (bl::format (std::string{_("Date field can not be empty if 'Multi-split' option is unset.\n")}) % + std::string{_(gnc_csv_col_type_strs[prop_type])}).str()); break; case GncTransPropType::NUM: @@ -256,6 +240,10 @@ void GncPreTrans::set (GncTransPropType prop_type, const std::string& value) m_desc = boost::none; if (!value.empty()) m_desc = value; + else if (!m_multi_split) + throw std::invalid_argument ( + (bl::format (std::string{_("Description field can not be empty if 'Multi-split' option is unset.\n")}) % + std::string{_(gnc_csv_col_type_strs[prop_type])}).str()); break; case GncTransPropType::NOTES: @@ -325,7 +313,7 @@ std::string GncPreTrans::verify_essentials (void) return std::string(); } -Transaction* GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency) +std::shared_ptr GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency) { if (created) return nullptr; @@ -357,7 +345,7 @@ Transaction* GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency) created = true; - return trans; + return std::make_shared(trans); } bool GncPreTrans::is_part_of (std::shared_ptr parent) @@ -427,8 +415,8 @@ void GncPreSplit::set (GncTransPropType prop_type, const std::string& value) m_account = boost::none; if (value.empty()) throw std::invalid_argument (_("Account value can't be empty.")); - acct = gnc_csv_account_map_search (value.c_str()); - if (acct) + if ((acct = gnc_csv_account_map_search (value.c_str())) || + (acct = gnc_account_lookup_by_full_name (gnc_get_current_root_account(), value.c_str()))) m_account = acct; else throw std::invalid_argument (_(bad_acct)); @@ -439,8 +427,8 @@ void GncPreSplit::set (GncTransPropType prop_type, const std::string& value) if (value.empty()) throw std::invalid_argument (_("Transfer account value can't be empty.")); - acct = gnc_csv_account_map_search (value.c_str()); - if (acct) + if ((acct = gnc_csv_account_map_search (value.c_str())) || + (acct = gnc_account_lookup_by_full_name (gnc_get_current_root_account(), value.c_str()))) m_taccount = acct; else throw std::invalid_argument (_(bad_tacct)); @@ -458,18 +446,32 @@ void GncPreSplit::set (GncTransPropType prop_type, const std::string& value) m_tmemo = value; break; - case GncTransPropType::DEPOSIT: - m_deposit = boost::none; - m_deposit = parse_amount (value, m_currency_format); // Will throw if parsing fails + case GncTransPropType::AMOUNT: + m_amount = boost::none; + m_amount = parse_monetary (value, m_currency_format); // Will throw if parsing fails break; - case GncTransPropType::WITHDRAWAL: - m_withdrawal = boost::none; - m_withdrawal = parse_amount (value, m_currency_format); // Will throw if parsing fails + + case GncTransPropType::AMOUNT_NEG: + m_amount_neg = boost::none; + m_amount_neg = parse_monetary (value, m_currency_format); // Will throw if parsing fails + break; + + case GncTransPropType::T_AMOUNT: + m_tamount = boost::none; + m_tamount = parse_monetary (value, m_currency_format); // Will throw if parsing fails + break; + + case GncTransPropType::T_AMOUNT_NEG: + m_tamount_neg = boost::none; + m_tamount_neg = parse_monetary (value, m_currency_format); // Will throw if parsing fails break; case GncTransPropType::PRICE: + /* Note while a price is not stricly a currency, it will likely use + * the same decimal point as currencies in the csv file, so parse + * using the same parser */ m_price = boost::none; - m_price = parse_price (value); // Will throw if parsing fails + m_price = parse_monetary (value, m_currency_format); // Will throw if parsing fails break; case GncTransPropType::REC_STATE: @@ -545,18 +547,32 @@ void GncPreSplit::add (GncTransPropType prop_type, const std::string& value) auto num_val = GncNumeric(); switch (prop_type) { - case GncTransPropType::DEPOSIT: - num_val = parse_amount (value, m_currency_format); // Will throw if parsing fails - if (m_deposit) - num_val += *m_deposit; - m_deposit = num_val; + case GncTransPropType::AMOUNT: + num_val = parse_monetary (value, m_currency_format); // Will throw if parsing fails + if (m_amount) + num_val += *m_amount; + m_amount = num_val; break; - case GncTransPropType::WITHDRAWAL: - num_val = parse_amount (value, m_currency_format); // Will throw if parsing fails - if (m_withdrawal) - num_val += *m_withdrawal; - m_withdrawal = num_val; + case GncTransPropType::AMOUNT_NEG: + num_val = parse_monetary (value, m_currency_format); // Will throw if parsing fails + if (m_amount_neg) + num_val += *m_amount_neg; + m_amount_neg = num_val; + break; + + case GncTransPropType::T_AMOUNT: + num_val = parse_monetary (value, m_currency_format); // Will throw if parsing fails + if (m_tamount) + num_val += *m_tamount; + m_tamount = num_val; + break; + + case GncTransPropType::T_AMOUNT_NEG: + num_val = parse_monetary (value, m_currency_format); // Will throw if parsing fails + if (m_tamount_neg) + num_val += *m_tamount_neg; + m_tamount_neg = num_val; break; default: @@ -587,8 +603,8 @@ std::string GncPreSplit::verify_essentials (void) { auto err_msg = std::string(); /* Make sure this split has the minimum required set of properties defined. */ - if (!m_deposit && !m_withdrawal) - err_msg = _("No deposit or withdrawal column."); + if (!m_amount && !m_amount_neg) + err_msg = _("No amount or negated amount column."); if (m_rec_state && *m_rec_state == YREC && !m_rec_date) { @@ -611,53 +627,22 @@ std::string GncPreSplit::verify_essentials (void) * @param trans The transaction to add a split to * @param account The split's account * @param amount The split's amount + * @param value The split's value * @param rec_state The split's reconcile status * @param rec_date The split's reconcile date - * @param price The split's conversion rate from account commodity to transaction commodity */ -static void trans_add_split (Transaction* trans, Account* account, GncNumeric amount, +static void trans_add_split (Transaction* trans, Account* account, + GncNumeric amount, GncNumeric value, const boost::optional& action, const boost::optional& memo, const boost::optional& rec_state, - const boost::optional& rec_date, - const boost::optional price) + const boost::optional& rec_date) { QofBook* book = xaccTransGetBook (trans); auto split = xaccMallocSplit (book); xaccSplitSetAccount (split, account); xaccSplitSetParent (split, trans); xaccSplitSetAmount (split, static_cast(amount)); - auto trans_curr = xaccTransGetCurrency(trans); - auto acct_comm = xaccAccountGetCommodity(account); - GncNumeric value; - if (gnc_commodity_equiv(trans_curr, acct_comm)) - value = amount; - else if (price) - value = amount * *price; - else - { - auto time = xaccTransRetDatePosted (trans); - /* Import data didn't specify price, let's lookup the nearest in time */ - auto nprice = - gnc_pricedb_lookup_nearest_in_time64(gnc_pricedb_get_db(book), - acct_comm, trans_curr, time); - if (nprice) - { - /* Found a usable price. Let's check if the conversion direction is right */ - GncNumeric rate; - if (gnc_commodity_equiv(gnc_price_get_currency(nprice), trans_curr)) - rate = gnc_price_get_value(nprice); - else - rate = static_cast(gnc_price_get_value(nprice)).inv(); - - value = amount * rate; - } - else - { - PWARN("No price found, using a price of 1."); - value = amount; - } - } xaccSplitSetValue (split, static_cast(value)); if (memo) @@ -675,7 +660,7 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am } -void GncPreSplit::create_split (Transaction* trans) +void GncPreSplit::create_split (std::shared_ptr draft_trans) { if (created) return; @@ -692,33 +677,89 @@ void GncPreSplit::create_split (Transaction* trans) Account *account = nullptr; Account *taccount = nullptr; - auto deposit = GncNumeric(); - auto withdrawal = GncNumeric(); auto amount = GncNumeric(); if (m_account) account = *m_account; if (m_taccount) taccount = *m_taccount; - if (m_deposit) - deposit = *m_deposit; - if (m_withdrawal) - withdrawal = *m_withdrawal; - - amount = deposit - withdrawal; + if (m_amount) + amount += *m_amount; + if (m_amount_neg) + amount -= *m_amount_neg; /* Add a split with the cumulative amount value. */ - trans_add_split (trans, account, amount, m_action, m_memo, m_rec_state, m_rec_date, m_price); + // FIXME The first split is always assumed to be in the transaction currency, but this assumption + // may not hold in case of stock transactions. + // Needs extra testing. + auto inv_price = m_price; + if (m_price) + inv_price = m_price->inv(); + trans_add_split (draft_trans->trans, account, amount, amount, m_action, m_memo, m_rec_state, m_rec_date); if (taccount) { - /* Note: the current importer assumes at most 2 splits. This means the second split amount - * will be the negative of the first split amount. - */ - auto inv_price = m_price; - if (m_price) - inv_price = m_price->inv(); - trans_add_split (trans, taccount, -amount, m_taction, m_tmemo, m_trec_state, m_trec_date, inv_price); + /* If a taccount is set that forcibly means we're processing a single-line transaction + * Determine the transfer amount. If the csv data had columns for it, use those, otherwise + * assume transfer amount is */ + auto tamount = GncNumeric(); + auto tvalue = -tamount; + if (m_tamount || m_tamount_neg) + { + if (m_tamount) + tamount += *m_tamount; + if (m_tamount_neg) + tamount -= *m_tamount_neg; + tvalue = -amount; + } + else + { + auto trans_curr = xaccTransGetCurrency(draft_trans->trans); + auto acct_comm = xaccAccountGetCommodity(taccount); + if (gnc_commodity_equiv(trans_curr, acct_comm)) + tamount = -amount; + else if (m_price) + tamount = -amount * *m_price; + else + { + QofBook* book = xaccTransGetBook (draft_trans->trans); + auto time = xaccTransRetDatePosted (draft_trans->trans); + /* Import data didn't specify price, let's lookup the nearest in time */ + auto nprice = + gnc_pricedb_lookup_nearest_in_time64(gnc_pricedb_get_db(book), + acct_comm, trans_curr, time); + if (nprice) + { + /* Found a usable price. Let's check if the conversion direction is right */ + GncNumeric rate; + if (gnc_commodity_equiv(gnc_price_get_currency(nprice), trans_curr)) + rate = gnc_price_get_value(nprice); + else + rate = static_cast(gnc_price_get_value(nprice)).inv(); + + tamount = -amount * rate; + } + else + { + PWARN("No price found, not creating second split."); + /* Set bogus value for tamount so we can skip creation further down */ + // FIXME should somehow pass the account to generic import matcher + // so it can have a shot a asking for an exchange rate and + // creating the split properly + tamount = -tvalue; + } + } + } + if (tamount != -tvalue) + trans_add_split (draft_trans->trans, taccount, tamount, tvalue, m_taction, m_tmemo, m_trec_state, m_trec_date); + } + else + { + draft_trans->m_price = m_price; + draft_trans->m_taction = m_taction; + draft_trans->m_tmemo = m_tmemo; + draft_trans->m_trec_state = m_trec_state; + draft_trans->m_trec_date = m_trec_date; } created = true; diff --git a/gnucash/import-export/csv-imp/gnc-imp-props-tx.hpp b/gnucash/import-export/csv-imp/gnc-imp-props-tx.hpp index 5999fb7257..d9f61d0528 100644 --- a/gnucash/import-export/csv-imp/gnc-imp-props-tx.hpp +++ b/gnucash/import-export/csv-imp/gnc-imp-props-tx.hpp @@ -59,18 +59,20 @@ enum class GncTransPropType { ACTION, ACCOUNT, - DEPOSIT, - WITHDRAWAL, + AMOUNT, + AMOUNT_NEG, PRICE, MEMO, REC_STATE, REC_DATE, TACTION, TACCOUNT, + T_AMOUNT, + T_AMOUNT_NEG, TMEMO, TREC_STATE, TREC_DATE, - SPLIT_PROPS = TMEMO + SPLIT_PROPS = TREC_DATE }; /** Maps all column types to a string representation. @@ -103,18 +105,49 @@ GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split); gnc_commodity* parse_commodity (const std::string& comm_str); -GncNumeric parse_amount (const std::string &str, int currency_format); +GncNumeric parse_monetary (const std::string &str, int currency_format); + + +/** The final form of a transaction to import before it is passed on to the + * generic importer. + * + * @param trans a possibly incomplete transaction created based on the data + * collected from the PreTrans and PreSplit records + * + * @param m_price... values harvested from the import data in single + * line mode and for which the transfer split could not yet + * be created (due to a missing transfer account value). These + * parameters will be passed on to the generic importer + * which can use this to complete information on the balancing + * split for an incomplete transaction + */ +struct DraftTransaction +{ + DraftTransaction (Transaction* tx) : trans(tx) {} + ~DraftTransaction () { if (trans) { xaccTransDestroy (trans); trans = nullptr; } } + Transaction* trans; + + boost::optional m_price; + boost::optional m_taction; + boost::optional m_tmemo; + boost::optional m_trec_state; + boost::optional m_trec_date; + + boost::optional void_reason; +}; struct GncPreTrans { public: - GncPreTrans(int date_format) : m_date_format{date_format} {}; + GncPreTrans(int date_format, bool multi_split) + : m_date_format{date_format}, m_multi_split{multi_split} {}; void set (GncTransPropType prop_type, const std::string& value); void set_date_format (int date_format) { m_date_format = date_format ;} + void set_multi_split (bool multi_split) { m_multi_split = multi_split ;} void reset (GncTransPropType prop_type); std::string verify_essentials (void); - Transaction *create_trans (QofBook* book, gnc_commodity* currency); + std::shared_ptr create_trans (QofBook* book, gnc_commodity* currency); /** Check whether the harvested transaction properties for this instance * match those of another one (the "parent"). Note this function is *not* @@ -136,6 +169,7 @@ public: private: int m_date_format; + bool m_multi_split; boost::optional m_differ; boost::optional m_date; boost::optional m_num; @@ -159,7 +193,7 @@ public: void set_date_format (int date_format) { m_date_format = date_format ;} void set_currency_format (int currency_format) { m_currency_format = currency_format; } std::string verify_essentials (void); - void create_split(Transaction* trans); + void create_split(std::shared_ptr draft_trans); Account* get_account () { if (m_account) return *m_account; else return nullptr; } void set_account (Account* acct) { if (acct) m_account = acct; else m_account = boost::none; } @@ -170,14 +204,16 @@ private: int m_currency_format; boost::optional m_action; boost::optional m_account; - boost::optional m_deposit; - boost::optional m_withdrawal; + boost::optional m_amount; + boost::optional m_amount_neg; boost::optional m_price; boost::optional m_memo; boost::optional m_rec_state; boost::optional m_rec_date; boost::optional m_taction; boost::optional m_taccount; + boost::optional m_tamount; + boost::optional m_tamount_neg; boost::optional m_tmemo; boost::optional m_trec_state; boost::optional m_trec_date; diff --git a/gnucash/import-export/csv-imp/gnc-imp-settings-csv-tx.cpp b/gnucash/import-export/csv-imp/gnc-imp-settings-csv-tx.cpp index c867605372..a2e503704a 100644 --- a/gnucash/import-export/csv-imp/gnc-imp-settings-csv-tx.cpp +++ b/gnucash/import-export/csv-imp/gnc-imp-settings-csv-tx.cpp @@ -89,7 +89,7 @@ static std::shared_ptr create_int_gnc_exp_preset(void) GncTransPropType::ACCOUNT, GncTransPropType::NONE, GncTransPropType::NONE, - GncTransPropType::DEPOSIT, + GncTransPropType::AMOUNT, GncTransPropType::REC_STATE, GncTransPropType::REC_DATE, GncTransPropType::PRICE @@ -214,8 +214,16 @@ CsvTransImpSettings::load (void) &list_len, &key_error); for (uint32_t i = 0; i < list_len; i++) { + /* Special case a few legacy column names */ + const char *col_type_str = col_types_str[i]; + if (!g_strcmp0(col_type_str, "Deposit")) // -> "Amount" + col_type_str = gnc_csv_col_type_strs[GncTransPropType::AMOUNT]; + if (!g_strcmp0(col_type_str, "Withdrawal")) // -> "Amount (Negated)" + col_type_str = gnc_csv_col_type_strs[GncTransPropType::AMOUNT_NEG]; + if (!g_strcmp0(col_type_str, "Num")) // -> "Number" + col_type_str = gnc_csv_col_type_strs[GncTransPropType::NUM]; auto col_types_it = std::find_if (gnc_csv_col_type_strs.begin(), - gnc_csv_col_type_strs.end(), test_prop_type_str (col_types_str[i])); + gnc_csv_col_type_strs.end(), test_prop_type_str (col_type_str)); if (col_types_it != gnc_csv_col_type_strs.end()) { /* Found a valid column type. Now check whether it is allowed diff --git a/gnucash/import-export/csv-imp/gnc-import-tx.cpp b/gnucash/import-export/csv-imp/gnc-import-tx.cpp index e70a1237c2..9bf4a1a3be 100644 --- a/gnucash/import-export/csv-imp/gnc-import-tx.cpp +++ b/gnucash/import-export/csv-imp/gnc-import-tx.cpp @@ -216,8 +216,8 @@ void GncTxImport::currency_format (int currency_format) m_settings.m_currency_format = currency_format; /* Reparse all currency related columns */ - std::vector commodities = { GncTransPropType::DEPOSIT, - GncTransPropType::WITHDRAWAL, + std::vector commodities = { GncTransPropType::AMOUNT, + GncTransPropType::AMOUNT_NEG, GncTransPropType::PRICE}; reset_formatted_column (commodities); } @@ -402,7 +402,7 @@ void GncTxImport::tokenize (bool guessColTypes) auto length = tokenized_line.size(); if (length > 0) m_parsed_lines.push_back (std::make_tuple (tokenized_line, std::string(), - std::make_shared(date_format()), + std::make_shared(date_format(), m_settings.m_multi_split), std::make_shared(date_format(), currency_format()), false)); if (length > max_cols) @@ -485,11 +485,11 @@ void GncTxImport::verify_column_selections (ErrorList& error_msg) if (!check_for_column_type(GncTransPropType::DESCRIPTION)) error_msg.add_error( _("Please select a description column.")); - /* Verify at least one amount column (deposit or withdrawal) column is selected. + /* Verify at least one amount column (amount or amount_neg) column is selected. */ - if (!check_for_column_type(GncTransPropType::DEPOSIT) && - !check_for_column_type(GncTransPropType::WITHDRAWAL)) - error_msg.add_error( _("Please select a deposit or withdrawal column.")); + if (!check_for_column_type(GncTransPropType::AMOUNT) && + !check_for_column_type(GncTransPropType::AMOUNT_NEG)) + error_msg.add_error( _("Please select a amount or negated amount column.")); /* Verify a transfer account is selected if any of the other transfer properties * are selected. @@ -604,9 +604,9 @@ std::shared_ptr GncTxImport::trans_properties_to_trans (std::v QofBook* book = gnc_account_get_book (account); gnc_commodity* currency = xaccAccountGetCommodity (account); - auto trans = trans_props->create_trans (book, currency); + auto draft_trans = trans_props->create_trans (book, currency); - if (trans) + if (draft_trans) { /* We're about to continue with a new transaction * Time to do some closing actions on the previous one @@ -621,19 +621,22 @@ std::shared_ptr GncTxImport::trans_properties_to_trans (std::v xaccTransCommitEdit (m_current_draft->trans); xaccTransVoid (m_current_draft->trans, m_current_draft->void_reason->c_str()); } - m_current_draft = std::make_shared(trans); + m_current_draft = draft_trans; m_current_draft->void_reason = trans_props->get_void_reason(); created_trans = true; } else if (m_settings.m_multi_split) // in multi_split mode create_trans will return a nullptr for all but the first split - trans = m_current_draft->trans; + draft_trans = m_current_draft; else // in non-multi-split mode each line should be a transaction, so not having one here is an error throw std::invalid_argument ("Failed to create transaction from selected columns."); - if (!trans) + if (!draft_trans) return nullptr; - split_props->create_split(trans); + split_props->create_split (draft_trans); + + // With the added split information, we may have to revisit the transaction's commodity here + // TBD /* Only return the draft transaction if we really created a new one * The return value will be added to a list for further processing, @@ -737,93 +740,55 @@ GncTxImport::check_for_column_type (GncTransPropType type) } /* A helper function intended to be called only from set_column_type */ -void GncTxImport::update_pre_trans_props (uint32_t row, uint32_t col, GncTransPropType prop_type) +void GncTxImport::update_pre_trans_split_props (uint32_t row, uint32_t col, GncTransPropType old_type, GncTransPropType new_type) { - if ((prop_type == GncTransPropType::NONE) || (prop_type > GncTransPropType::TRANS_PROPS)) - return; /* Only deal with transaction related properties. */ - /* Deliberately make a copy of the GncPreTrans. It may be the original one was shared - * with a previous line and should no longer be after the transprop is changed. */ + * with a previous line and should no longer be after the transprop is changed. + * This doesn't apply for the GncPreSplit so we just get a pointer to it for easier processing. + */ auto trans_props = std::make_shared (*(std::get(m_parsed_lines[row])).get()); - auto value = std::string(); - - if (col < std::get(m_parsed_lines[row]).size()) - value = std::get(m_parsed_lines[row]).at(col); - - if (value.empty()) - trans_props->reset (prop_type); - else - { - try - { - trans_props->set(prop_type, value); - } - catch (const std::exception& e) - { - /* Do nothing, just prevent the exception from escalating up - * However log the error if it happens on a row that's not skipped - */ - if (!std::get(m_parsed_lines[row])) - PINFO("User warning: %s", e.what()); - } - } - - /* Store the result */ - std::get(m_parsed_lines[row]) = trans_props; - - /* For multi-split input data, we need to check whether this line is part of - * a transaction that has already been started by a previous line. */ - if (m_settings.m_multi_split) - { - if (trans_props->is_part_of(m_parent)) - { - /* This line is part of an already started transaction - * continue with that one instead to make sure the split from this line - * gets added to the proper transaction */ - std::get(m_parsed_lines[row]) = m_parent; - } - else - { - /* This line starts a new transaction, set it as parent for - * subsequent lines. */ - m_parent = trans_props; - } - } -} - -/* A helper function intended to be called only from set_column_type */ -void GncTxImport::update_pre_split_props (uint32_t row, uint32_t col, GncTransPropType prop_type) -{ - if ((prop_type > GncTransPropType::SPLIT_PROPS) || (prop_type <= GncTransPropType::TRANS_PROPS)) - return; /* Only deal with split related properties. */ - auto split_props = std::get(m_parsed_lines[row]); - split_props->reset (prop_type); + /* Start by resetting the value of the old_type. */ + if ((old_type > GncTransPropType::NONE) && (old_type <= GncTransPropType::TRANS_PROPS)) + trans_props->reset (old_type); + if ((old_type > GncTransPropType::TRANS_PROPS) && (old_type <= GncTransPropType::SPLIT_PROPS)) + split_props->reset (old_type); + + /* Next attempt to set the value for new_type (may throw!) */ try { - // Except for Deposit or Withdrawal lines there can only be - // one column with a given property type. - if ((prop_type != GncTransPropType::DEPOSIT) && - (prop_type != GncTransPropType::WITHDRAWAL)) + if ((new_type > GncTransPropType::NONE) && (new_type <= GncTransPropType::TRANS_PROPS)) { - auto value = std::get(m_parsed_lines[row]).at(col); - split_props->set(prop_type, value); + auto value = std::string(); + + if (col < std::get(m_parsed_lines[row]).size()) + value = std::get(m_parsed_lines[row]).at(col); + + trans_props->set(new_type, value); } - else + + if ((new_type > GncTransPropType::TRANS_PROPS) && (new_type <= GncTransPropType::SPLIT_PROPS)) { - // For Deposits and Withdrawal we have to sum all columns with this property - for (auto col_it = m_settings.m_column_types.cbegin(); - col_it < m_settings.m_column_types.cend(); - col_it++) + /* Except for Deposit or Withdrawal lines there can only be + * one column with a given property type. */ + if ((new_type != GncTransPropType::AMOUNT) && + (new_type != GncTransPropType::AMOUNT_NEG)) { - if (*col_it == prop_type) - { - auto col_num = col_it - m_settings.m_column_types.cbegin(); - auto value = std::get(m_parsed_lines[row]).at(col_num); - split_props->add (prop_type, value); - } + auto value = std::get(m_parsed_lines[row]).at(col); + split_props->set(new_type, value); } + else + /* For Deposits and Withdrawal we have to sum all columns with this property */ + for (auto col_it = m_settings.m_column_types.cbegin(); + col_it < m_settings.m_column_types.cend(); + col_it++) + if (*col_it == new_type) + { + auto col_num = col_it - m_settings.m_column_types.cbegin(); + auto value = std::get(m_parsed_lines[row]).at(col_num); + split_props->add (new_type, value); + } } } catch (const std::exception& e) @@ -834,6 +799,24 @@ void GncTxImport::update_pre_split_props (uint32_t row, uint32_t col, GncTransPr if (!std::get(m_parsed_lines[row])) PINFO("User warning: %s", e.what()); } + + /* All value updates are finished by now, time to determine + * what to do with the updated GncPreTrans copy. + * + * For multi-split input data this line may be part of a transaction + * that has already been started by a previous line. In that case + * reuse the GncPreTrans from that previous line (which we track + * in m_parent). + * In all other cases our new GncPreTrans should be used for this line + * and be marked as the new potential m_parent for subsequent lines. + */ + if (m_settings.m_multi_split && trans_props->is_part_of(m_parent)) + std::get(m_parsed_lines[row]) = m_parent; + else + { + std::get(m_parsed_lines[row]) = trans_props; + m_parent = trans_props; + } } @@ -847,10 +830,10 @@ GncTxImport::set_column_type (uint32_t position, GncTransPropType type, bool for if ((type == old_type) && !force) return; /* Nothing to do */ - // Column types except deposit and withdrawal should be unique, + // Column types except amount and negated amount should be unique, // so remove any previous occurrence of the new type - if ((type != GncTransPropType::DEPOSIT) && - (type != GncTransPropType::WITHDRAWAL)) + if ((type != GncTransPropType::AMOUNT) && + (type != GncTransPropType::AMOUNT_NEG)) std::replace(m_settings.m_column_types.begin(), m_settings.m_column_types.end(), type, GncTransPropType::NONE); @@ -870,32 +853,14 @@ GncTxImport::set_column_type (uint32_t position, GncTransPropType type, bool for * to ensure column updates use the most recent one */ std::get(*parsed_lines_it)->set_date_format (m_settings.m_date_format); + std::get(*parsed_lines_it)->set_multi_split (m_settings.m_multi_split); std::get(*parsed_lines_it)->set_date_format (m_settings.m_date_format); std::get(*parsed_lines_it)->set_currency_format (m_settings.m_currency_format); uint32_t row = parsed_lines_it - m_parsed_lines.begin(); - /* If the column type actually changed, first reset the property - * represented by the old column type - */ - if (old_type != type) - { - auto old_col = std::get(*parsed_lines_it).size(); // Deliberately out of bounds to trigger a reset! - if ((old_type > GncTransPropType::NONE) - && (old_type <= GncTransPropType::TRANS_PROPS)) - update_pre_trans_props (row, old_col, old_type); - else if ((old_type > GncTransPropType::TRANS_PROPS) - && (old_type <= GncTransPropType::SPLIT_PROPS)) - update_pre_split_props (row, old_col, old_type); - } - - /* Then set the property represented by the new column type */ - if ((type > GncTransPropType::NONE) - && (type <= GncTransPropType::TRANS_PROPS)) - update_pre_trans_props (row, position, type); - else if ((type > GncTransPropType::TRANS_PROPS) - && (type <= GncTransPropType::SPLIT_PROPS)) - update_pre_split_props (row, position, type); + /* Update the property represented by the new column type */ + update_pre_trans_split_props (row, position, old_type, type); /* Report errors if there are any */ auto trans_errors = std::get(*parsed_lines_it)->errors(); @@ -904,7 +869,6 @@ GncTxImport::set_column_type (uint32_t position, GncTransPropType type, bool for trans_errors + (trans_errors.empty() && split_errors.empty() ? std::string() : "\n") + split_errors; - } } diff --git a/gnucash/import-export/csv-imp/gnc-import-tx.hpp b/gnucash/import-export/csv-imp/gnc-import-tx.hpp index aa3e881c6d..feb4860cb4 100644 --- a/gnucash/import-export/csv-imp/gnc-import-tx.hpp +++ b/gnucash/import-export/csv-imp/gnc-import-tx.hpp @@ -45,19 +45,6 @@ #include -/** This struct stores a possibly incomplete transaction - * optionally together with its intended balance in case - * the user had selected a balance column. */ -struct DraftTransaction -{ - DraftTransaction (Transaction* tx) : trans(tx), balance(gnc_numeric_zero()), balance_set(false) {} - ~DraftTransaction () { if (trans) { xaccTransDestroy (trans); trans = nullptr; } } - Transaction* trans; - gnc_numeric balance; /**< The expected balance after this transaction takes place */ - bool balance_set; /**< true if balance has been set from user data, false otherwise */ - boost::optional void_reason; -}; - /* A set of currency formats that the user sees. */ extern const int num_currency_formats; extern const gchar* currency_format_user[]; @@ -181,11 +168,10 @@ private: */ std::shared_ptr trans_properties_to_trans (std::vector::iterator& parsed_line); - /* Two internal helper functions that should only be called from within + /* Internal helper function that should only be called from within * set_column_type for consistency (otherwise error messages may not be (re)set) */ - void update_pre_trans_props (uint32_t row, uint32_t col, GncTransPropType prop_type); - void update_pre_split_props (uint32_t row, uint32_t col, GncTransPropType prop_type); + void update_pre_trans_split_props (uint32_t row, uint32_t col, GncTransPropType old_type, GncTransPropType new_type); struct CsvTranImpSettings; //FIXME do we need this line CsvTransImpSettings m_settings; diff --git a/gnucash/import-export/import-account-matcher.c b/gnucash/import-export/import-account-matcher.c index 00001e81d8..9add090ba5 100644 --- a/gnucash/import-export/import-account-matcher.c +++ b/gnucash/import-export/import-account-matcher.c @@ -72,26 +72,6 @@ partial_match_if_valid (AccountOnlineMatch *match) * Functions needed by gnc_import_select_account * \********************************************************************/ -/** Constructor for AccountPickerDialog. - * @return Pointer to a new AccountPickerDialog - */ -static AccountPickerDialog* gnc_import_new_account_picker(void) -{ - AccountPickerDialog* picker = g_new(AccountPickerDialog, 1); - picker->dialog = NULL; - picker->account_tree = NULL; - picker->account_tree_sw = NULL; - picker->auto_create = TRUE; - picker->account_human_description = NULL; - picker->account_online_id_value = NULL; - picker->account_online_id_label = NULL; - picker->new_account_default_commodity = NULL; - picker->new_account_default_type = 0; - picker->default_account = NULL; - picker->retAccount = NULL; - return picker; -} - /************************************************** * test_acct_online_id_match @@ -276,24 +256,6 @@ show_placeholder_warning (AccountPickerDialog *picker, const gchar *name) } -/*********************************************************** - * show_commodity_warning - * - * show the warning when account is a different commodity to that - * required - ************************************************************/ -static void -show_commodity_warning (AccountPickerDialog *picker, const gchar *name) -{ - const gchar *com_name = gnc_commodity_get_fullname (picker->new_account_default_commodity); - gchar *text = g_strdup_printf (_("The account '%s' has a different commodity to the " - "one required, '%s'. Please choose a different account."), - name, com_name); - - show_warning (picker, text); -} - - /******************************************************* * account_tree_row_changed_cb * @@ -305,31 +267,16 @@ account_tree_row_changed_cb (GtkTreeSelection *selection, { Account *sel_account = gnc_tree_view_account_get_selected_account (picker->account_tree); - if (!sel_account) - { - gtk_widget_hide (GTK_WIDGET(picker->whbox)); // hide the warning - gtk_widget_set_sensitive (picker->ok_button, FALSE); // disable OK button - return; - } - - gtk_widget_set_sensitive (picker->ok_button, TRUE); // enable OK button + /* Reset buttons and warnings */ + gtk_widget_hide (GTK_WIDGET(picker->whbox)); + gtk_widget_set_sensitive (picker->ok_button, (sel_account != NULL)); /* See if the selected account is a placeholder. */ if (sel_account && xaccAccountGetPlaceholder (sel_account)) { const gchar *retval_name = xaccAccountGetName (sel_account); - show_placeholder_warning (picker, retval_name); } - else if (picker->new_account_default_commodity && - (!gnc_commodity_equal (xaccAccountGetCommodity (sel_account), - picker->new_account_default_commodity))) // check commodity - { - const gchar *retval_name = xaccAccountGetName (sel_account); - show_commodity_warning (picker, retval_name); - } - else - gtk_widget_hide (GTK_WIDGET(picker->whbox)); // hide the warning } static gboolean @@ -402,7 +349,7 @@ account_tree_row_activated_cb(GtkTreeView *view, GtkTreePath *path, *******************************************************/ Account * gnc_import_select_account(GtkWidget *parent, const gchar * account_online_id_value, - gboolean auto_create, + gboolean prompt_on_no_match, const gchar * account_human_description, const gnc_commodity * new_account_default_commodity, GNCAccountType new_account_default_type, @@ -424,13 +371,12 @@ Account * gnc_import_select_account(GtkWidget *parent, DEBUG("Default account type received: %s", xaccAccountGetTypeStr( new_account_default_type)); picker = g_new0(AccountPickerDialog, 1); - picker->account_online_id_value = account_online_id_value; picker->account_human_description = account_human_description; picker->new_account_default_commodity = new_account_default_commodity; picker->new_account_default_type = new_account_default_type; /*DEBUG("Looking for account with online_id: \"%s\"", account_online_id_value);*/ - if (account_online_id_value != NULL) + if (account_online_id_value) { AccountOnlineMatch match = {NULL, 0, account_online_id_value}; retval = @@ -441,7 +387,7 @@ Account * gnc_import_select_account(GtkWidget *parent, new_account_default_type == ACCT_TYPE_NONE) retval = match.partial_match; } - if (retval == NULL && auto_create != 0) + if (!retval && prompt_on_no_match) { /* load the interface */ builder = gtk_builder_new(); @@ -517,16 +463,13 @@ Account * gnc_import_select_account(GtkWidget *parent, case GTK_RESPONSE_OK: retval = gnc_tree_view_account_get_selected_account(picker->account_tree); - if (retval == NULL) + if (!retval) { response = GNC_RESPONSE_NEW; break; } - if (retval) - retval_name = xaccAccountGetName(retval); - if (!retval_name) - retval_name = "(null)"; - DEBUG("Selected account %p, %s", retval, retval_name); + retval_name = xaccAccountGetName(retval); + DEBUG("Selected account %p, %s", retval, retval_name ? retval_name : "(null)"); /* See if the selected account is a placeholder. */ if (retval && xaccAccountGetPlaceholder (retval)) @@ -535,16 +478,8 @@ Account * gnc_import_select_account(GtkWidget *parent, response = GNC_RESPONSE_NEW; break; } - else if (picker->new_account_default_commodity && - (!gnc_commodity_equal (xaccAccountGetCommodity (retval), - picker->new_account_default_commodity))) // check commodity - { - show_commodity_warning (picker, retval_name); - response = GNC_RESPONSE_NEW; - break; - } - if ( account_online_id_value != NULL) + if (account_online_id_value) { gnc_import_set_acc_online_id(retval, account_online_id_value); } diff --git a/gnucash/import-export/import-account-matcher.h b/gnucash/import-export/import-account-matcher.h index a7d9be613e..54e90f8eb4 100644 --- a/gnucash/import-export/import-account-matcher.h +++ b/gnucash/import-export/import-account-matcher.h @@ -42,18 +42,12 @@ extern "C" { typedef struct { GtkWidget *dialog; /* Dialog Widget */ - GtkWidget *new_button; /* new account button Widget */ GtkWidget *ok_button; /* ok button Widget */ GncTreeViewAccount *account_tree; /* Account tree */ GtkWidget *account_tree_sw; /* Scroll Window for Account tree */ - gboolean auto_create; /* Auto create retAccount, can be used to step over this stage */ const gchar *account_human_description; /* description for on line id, incoming */ - const gchar *account_online_id_value; /* On line id value, incoming */ - GtkWidget *account_online_id_label; /* the label Widget for the on line id, incoming */ const gnc_commodity *new_account_default_commodity; /* new account default commodity, incoming */ GNCAccountType new_account_default_type; /* new account default type, incoming */ - Account *default_account; /* default account for selection, incoming */ - Account *retAccount; /* Account value returned to caller */ GtkWidget *whbox; /* Warning HBox */ GtkWidget *warning; /* Warning Label */ } AccountPickerDialog; @@ -78,7 +72,7 @@ typedef struct remembered elsewhere. You would fill account_human_description to tell the user what he is looking for. In this mode, the online_id field of the found account will not be touched. To use this mode, - auto_create must NOT be set to 0. + prompt_on_no_match must NOT be set to 0. @param account_human_description A human-readable description of @@ -100,11 +94,10 @@ typedef struct ACCT_TYPE_NONE, the function will also warn the user if the found or created account's commodity doesn't match. - @param auto_create - Only active if no account with the + @param prompt_on_no_match Only active if no account with the account_online_id_value could be found in gnucash, or if online-id - was NULL. In that case, if auto_create is TRUE (nonzero), the user - will be asked to create a new account. If auto_create is FALSE + was NULL. In that case, if prompt_on_no_match is TRUE (nonzero), the user + will be asked to create a new account. If prompt_on_no_match is FALSE (zero), this function will simply return NULL but will neither select nor create any account. @@ -121,7 +114,7 @@ typedef struct */ Account * gnc_import_select_account(GtkWidget *parent, const gchar * account_online_id_value, - gboolean auto_create, + gboolean prompt_on_no_match, const gchar * account_human_description, const gnc_commodity * new_account_default_commodity, GNCAccountType new_account_default_type, diff --git a/gnucash/import-export/import-backend.c b/gnucash/import-export/import-backend.c deleted file mode 100644 index b88ab6b64f..0000000000 --- a/gnucash/import-export/import-backend.c +++ /dev/null @@ -1,1376 +0,0 @@ -/********************************************************************\ - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License as * - * published by the Free Software Foundation; either version 2 of * - * the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License* - * along with this program; if not, contact: * - * * - * Free Software Foundation Voice: +1-617-542-5942 * - * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 * - * Boston, MA 02110-1301, USA gnu@gnu.org * -\********************************************************************/ -/** @addtogroup Import_Export - @{ */ -/** @internal - @file import-backend.c - @brief import-backend.c: Generic importer backend implementation (duplicate matching algorithm, action handling, etc.) - @author Copyright (C) 2002 Benoit GrĂ©goire - @author Christian Stimming - @author Copyright (c) 2006 David Hampton -*/ - -#include - -#include -#include -#include -#include - -#include - -#include "import-backend.h" -#include "import-utilities.h" -#include "Account.h" -#include "Query.h" -#include "gnc-engine.h" -#include "engine-helpers.h" -#include "gnc-prefs.h" -#include "gnc-ui-util.h" - -#define GNCIMPORT_DESC "desc" -#define GNCIMPORT_MEMO "memo" -#define GNCIMPORT_PAYEE "payee" - -/********************************************************************\ - * Constants * -\********************************************************************/ - -static QofLogModule log_module = GNC_MOD_IMPORT; - -/********************************************************************\ - * Forward declared prototypes * -\********************************************************************/ - -static void -matchmap_store_destination (GncImportMatchMap *matchmap, - GNCImportTransInfo *trans_info, - gboolean use_match); - - -/********************************************************************\ - * Structures passed between the functions * -\********************************************************************/ - -struct _selected_match_info -{ - GNCImportMatchInfo *selected_match; - gboolean selected_manually; -}; - -struct _transactioninfo -{ - Transaction * trans; - Split * first_split; - - /* GList of GNCImportMatchInfo's, one for each possible duplicate match. */ - GList * match_list; - GNCImportSelectedMatchInfo selected_match_info; - - GNCImportAction action; - GNCImportAction previous_action; - - /* A list of tokenized strings to use for bayesian matching purposes */ - GList * match_tokens; - - /* In case of a single destination account it is stored here. */ - Account *dest_acc; - gboolean dest_acc_selected_manually; - - /* Reference id to link gnc transaction to external object. E.g. aqbanking job id. */ - guint32 ref_id; - - /* When updating a matched transaction, append Description and Notes instead of replacing */ - gboolean append_text; -}; - -/* Some simple getters and setters for the above data types. */ - -GList * -gnc_import_TransInfo_get_match_list (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->match_list; -} - -void -gnc_import_TransInfo_set_match_list (GNCImportTransInfo *info, GList* match_list) -{ - g_assert (info); - info->match_list = match_list; - if (match_list) - { - info->selected_match_info.selected_match = match_list->data; - } - else - { - info->selected_match_info.selected_match = NULL; - gnc_import_TransInfo_set_action (info, GNCImport_ADD); - } -} - -Transaction * -gnc_import_TransInfo_get_trans (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->trans; -} - -gboolean -gnc_import_TransInfo_is_balanced (const GNCImportTransInfo *info) -{ - g_assert (info); - /* Assume that the importer won't create a transaction that involves two or more - currencies and no non-currency commodity. In that case can use the simpler - value imbalance check. */ - if (gnc_numeric_zero_p(xaccTransGetImbalanceValue(gnc_import_TransInfo_get_trans(info)))) - { - return TRUE; - } - else - { - return FALSE; - } -} - -Split * -gnc_import_TransInfo_get_fsplit (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->first_split; -} - -GNCImportMatchInfo * -gnc_import_TransInfo_get_selected_match (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->selected_match_info.selected_match; -} - -void -gnc_import_TransInfo_set_selected_match_info (GNCImportTransInfo *info, - GNCImportMatchInfo *match, - gboolean selected_manually) -{ - g_assert (info); - info->selected_match_info.selected_match = match; - info->selected_match_info.selected_manually = selected_manually; -} - -gboolean -gnc_import_TransInfo_get_match_selected_manually (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->selected_match_info.selected_manually; -} - -GNCImportAction -gnc_import_TransInfo_get_action (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->action; -} - -void -gnc_import_TransInfo_set_action (GNCImportTransInfo *info, - GNCImportAction action) -{ - g_assert (info); - if (action != info->action) - { - info->previous_action = info->action; - info->action = action; - } -} - -Account * -gnc_import_TransInfo_get_destacc (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->dest_acc; -} -void gnc_import_TransInfo_set_destacc (GNCImportTransInfo *info, - Account *acc, - gboolean selected_manually) -{ - g_assert (info); - info->dest_acc = acc; - info->dest_acc_selected_manually = selected_manually; - - /* Store the mapping to the other account in the MatchMap. */ - if (selected_manually) - { - matchmap_store_destination (NULL, info, FALSE); - } -} - -gboolean -gnc_import_TransInfo_get_destacc_selected_manually (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->dest_acc_selected_manually; -} - -guint32 -gnc_import_TransInfo_get_ref_id (const GNCImportTransInfo *info) -{ - g_assert (info); - return info->ref_id; -} - -void -gnc_import_TransInfo_set_ref_id (GNCImportTransInfo *info, - guint32 ref_id) -{ - g_assert (info); - info->ref_id = ref_id; -} - - -void -gnc_import_TransInfo_set_append_text (GNCImportTransInfo *info, - gboolean append_text) -{ - g_assert (info); - info->append_text = append_text; -} - - -Split * -gnc_import_MatchInfo_get_split (const GNCImportMatchInfo * info) -{ - g_assert (info); - return info->split; -} - -gint -gnc_import_MatchInfo_get_probability (const GNCImportMatchInfo * info) -{ - if (info) - { - return info->probability; - } - else - { - return 0; - } -} - -void gnc_import_TransInfo_delete (GNCImportTransInfo *info) -{ - if (info) - { - g_list_free (info->match_list); - /*If the transaction exists and is still open, it must be destroyed*/ - if (info->trans && xaccTransIsOpen(info->trans)) - { - xaccTransDestroy(info->trans); - xaccTransCommitEdit(info->trans); - } - if (info->match_tokens) - { - GList *node; - - for (node = info->match_tokens; node; node = node->next) - g_free (node->data); - - g_list_free (info->match_tokens); - } - g_free(info); - } -} - -GdkPixbuf* gen_probability_pixbuf(gint score_original, GNCImportSettings *settings, GtkWidget * widget) -{ - GdkPixbuf* retval = NULL; - gint i, j; - gint score; - const gint height = 15; - const gint width_each_bar = 7; - gchar * green_bar = ("bggggb "); - gchar * yellow_bar = ("byyyyb "); - gchar * red_bar = ("brrrrb "); - gchar * black_bar = ("bbbbbb "); - const gint width_first_bar = 1; - gchar * black_first_bar = ("b"); - const gint num_colors = 5; - gchar * size_str; - gchar * none_color_str = g_strdup_printf(" c None"); - gchar * green_color_str = g_strdup_printf("g c green"); - gchar * yellow_color_str = g_strdup_printf("y c yellow"); - gchar * red_color_str = g_strdup_printf("r c red"); - gchar * black_color_str = g_strdup_printf("b c black"); - gchar * xpm[2+num_colors+height]; - gint add_threshold, clear_threshold; - - g_assert(settings); - g_assert(widget); - if (score_original < 0) - { - score = 0; - } - else - { - score = score_original; - } - size_str = g_strdup_printf("%d%s%d%s%d%s", (width_each_bar * score) + width_first_bar/*width*/, " ", height, " ", num_colors, " 1"/*characters per pixel*/); - - /*DEBUG("Begin");*/ - xpm[0] = size_str; - xpm[1] = none_color_str; - xpm[2] = green_color_str; - xpm[3] = yellow_color_str; - xpm[4] = red_color_str; - xpm[5] = black_color_str; - add_threshold = gnc_import_Settings_get_add_threshold(settings); - clear_threshold = gnc_import_Settings_get_clear_threshold(settings); - - for (i = 0; i < height; i++) - { - xpm[num_colors+1+i] = g_new0(char, (width_each_bar * score) + width_first_bar + 1); - for (j = 0; j <= score; j++) - { - if (i == 0 || i == height - 1) - { - if (j == 0) - { - strcat(xpm[num_colors+1+i], black_first_bar); - } - else - { - strcat(xpm[num_colors+1+i], black_bar); - } - } - else - { - if (j == 0) - { - strcat(xpm[num_colors+1+i], black_first_bar); - } - else if (j <= add_threshold) - { - strcat(xpm[num_colors+1+i], red_bar); - } - else if (j >= clear_threshold) - { - strcat(xpm[num_colors+1+i], green_bar); - } - else - { - strcat(xpm[num_colors+1+i], yellow_bar); - } - } - } - } - - retval = gdk_pixbuf_new_from_xpm_data((const gchar **)xpm); - for (i = 0; i <= num_colors + height; i++) - { - /*DEBUG("free_loop i=%d%s%s",i,": ",xpm[i]);*/ - g_free(xpm[i]); - } - - return retval; -} - -/************************************************************************* - * MatchMap- related functions (storing and retrieving) - */ - -/* Tokenize a string and append to an existing GList(or an empty GList) - * the tokens - */ -static GList* -tokenize_string(GList* existing_tokens, const char *string) -{ - char **tokenized_strings; /* array of strings returned by g_strsplit() */ - char **stringpos; - - tokenized_strings = g_strsplit(string, " ", 0); - stringpos = tokenized_strings; - - /* add each token to the token GList */ - while (stringpos && *stringpos) - { - if (strlen(*stringpos) > 0) - { - /* check for duplicated tokens */ - gboolean duplicated = FALSE; - for (GList* token = existing_tokens; token != NULL; token = token->next) - { - if (g_strcmp0(token->data, *stringpos) == 0) - { - duplicated = TRUE; - break; - } - } - if (duplicated == FALSE) - { - /* prepend the char* to the token GList */ - existing_tokens = g_list_prepend(existing_tokens, g_strdup(*stringpos)); - } - } - - /* then move to the next string */ - stringpos++; - } - - /* free up the strings that g_strsplit() created */ - g_strfreev(tokenized_strings); - - return existing_tokens; -} - -/* create and return a list of tokens for a given transaction info. */ -static GList* -TransactionGetTokens(GNCImportTransInfo *info) -{ - Transaction* transaction; - GList* tokens; - const char* text; - time64 transtime; - struct tm *tm_struct; - char local_day_of_week[16]; - - g_return_val_if_fail (info, NULL); - if (info->match_tokens) return info->match_tokens; - - transaction = gnc_import_TransInfo_get_trans(info); - g_assert(transaction); - - tokens = 0; /* start off with an empty list */ - - /* make tokens from the transaction description */ - text = xaccTransGetDescription(transaction); - tokens = tokenize_string(tokens, text); - - /* The day of week the transaction occurred is a good indicator of - * what account this transaction belongs in. Get the date and convert - * it to day of week as a token - */ - transtime = xaccTransGetDate(transaction); - tm_struct = gnc_gmtime(&transtime); - if (!qof_strftime(local_day_of_week, sizeof(local_day_of_week), "%A", tm_struct)) - { - PERR("TransactionGetTokens: error, strftime failed\n"); - } - gnc_tm_free (tm_struct); - /* we cannot add a locally allocated string to this array, dup it so - * it frees the same way the rest do - */ - tokens = g_list_prepend(tokens, g_strdup(local_day_of_week)); - - /* make tokens from the memo of each split of this transaction */ - for (GList *split=xaccTransGetSplitList (transaction); split; split=split->next) - { - text = xaccSplitGetMemo(split->data); - tokens = tokenize_string(tokens, text); - } - - /* remember the list of tokens for later.. */ - info->match_tokens = tokens; - - /* return the pointer to the GList */ - return tokens; -} -/* Destroy an import map. But all stored entries will still continue - * to exist in the underlying kvp frame of the account. - */ -static void -gnc_imap_destroy (GncImportMatchMap *imap) -{ - if (!imap) return; - g_free (imap); -} - -/* searches using the GNCImportTransInfo through all existing transactions - * if there is an exact match of the description and memo - */ -static Account * -matchmap_find_destination (GncImportMatchMap *matchmap, GNCImportTransInfo *info) -{ - GncImportMatchMap *tmp_map; - Account *result; - GList* tokens; - gboolean useBayes; - - g_assert (info); - tmp_map = ((matchmap != NULL) ? matchmap : - gnc_account_imap_create_imap - (xaccSplitGetAccount - (gnc_import_TransInfo_get_fsplit (info)))); - - useBayes = gnc_prefs_get_bool (GNC_PREFS_GROUP_IMPORT, GNC_PREF_USE_BAYES); - if (useBayes) - { - /* get the tokens for this transaction* */ - tokens = TransactionGetTokens(info); - - /* try to find the destination account for this transaction from its tokens */ - result = gnc_account_imap_find_account_bayes(tmp_map, tokens); - - } - else - { - /* old system of transaction to account matching */ - result = gnc_account_imap_find_account - (tmp_map, GNCIMPORT_DESC, - xaccTransGetDescription (gnc_import_TransInfo_get_trans (info))); - } - - /* Disable matching by memo, until bayesian filtering is implemented. - * It's currently unlikely to help, and has adverse effects, - * causing false positives, since very often the type of the - * transaction is stored there. - - if (result == NULL) - result = gnc_account_imap_find_account - (tmp_map, GNCIMPORT_MEMO, - xaccSplitGetMemo (gnc_import_TransInfo_get_fsplit (info))); - */ - - if (matchmap == NULL) - gnc_imap_destroy (tmp_map); - - return result; -} - -/** Store the destination account from trans_info in the matchmap. If - 'use_match' is true, the destination account of the selected - matching/duplicate transaction is used; otherwise, the stored - destination_acc pointer is used. */ -static void -matchmap_store_destination (GncImportMatchMap *matchmap, - GNCImportTransInfo *trans_info, - gboolean use_match) -{ - GncImportMatchMap *tmp_matchmap = NULL; - Account *dest; - const char *descr, *memo; - GList *tokens; - gboolean useBayes; - - g_assert (trans_info); - - /* This will store the destination account of the selected match if - the reconcile match selected has only two splits. Good idea - Christian! */ - dest = ((use_match) ? - xaccSplitGetAccount - (xaccSplitGetOtherSplit - (gnc_import_MatchInfo_get_split - (gnc_import_TransInfo_get_selected_match (trans_info)))) : - gnc_import_TransInfo_get_destacc (trans_info)); - if (dest == NULL) - return; - - tmp_matchmap = ((matchmap != NULL) ? - matchmap : - gnc_account_imap_create_imap - (xaccSplitGetAccount - (gnc_import_TransInfo_get_fsplit (trans_info)))); - - /* see what matching system we are currently using */ - useBayes = gnc_prefs_get_bool (GNC_PREFS_GROUP_IMPORT, GNC_PREF_USE_BAYES); - if (useBayes) - { - /* tokenize this transaction */ - tokens = TransactionGetTokens(trans_info); - - /* add the tokens to the imap with the given destination account */ - gnc_account_imap_add_account_bayes(tmp_matchmap, tokens, dest); - - } - else - { - /* old matching system */ - descr = xaccTransGetDescription - (gnc_import_TransInfo_get_trans (trans_info)); - if (descr && (strlen (descr) > 0)) - gnc_account_imap_add_account (tmp_matchmap, - GNCIMPORT_DESC, - descr, - dest); - memo = xaccSplitGetMemo - (gnc_import_TransInfo_get_fsplit (trans_info)); - if (memo && (strlen (memo) > 0)) - gnc_account_imap_add_account (tmp_matchmap, - GNCIMPORT_MEMO, - memo, - dest); - } /* if(useBayes) */ - - if (matchmap == NULL) - gnc_imap_destroy (tmp_matchmap); -} - - - -/** @brief The transaction matching heuristics are here. - */ -void split_find_match (GNCImportTransInfo * trans_info, - Split * split, - gint display_threshold, - gint date_threshold, - gint date_not_threshold, - double fuzzy_amount_difference) -{ - /* DEBUG("Begin"); */ - - /*Ignore the split if the transaction is open for edit, meaning it - was just downloaded. */ - if (xaccTransIsOpen(xaccSplitGetParent(split)) == FALSE) - { - GNCImportMatchInfo * match_info; - gint prob = 0; - gboolean update_proposed; - double downloaded_split_amount, match_split_amount; - time64 match_time, download_time; - int datediff_day; - Transaction *new_trans = gnc_import_TransInfo_get_trans (trans_info); - Split *new_trans_fsplit = gnc_import_TransInfo_get_fsplit (trans_info); - - /* Matching heuristics */ - - /* Amount heuristics */ - downloaded_split_amount = - gnc_numeric_to_double (xaccSplitGetAmount(new_trans_fsplit)); - /*DEBUG(" downloaded_split_amount=%f", downloaded_split_amount);*/ - match_split_amount = gnc_numeric_to_double(xaccSplitGetAmount(split)); - /*DEBUG(" match_split_amount=%f", match_split_amount);*/ - if (fabs(downloaded_split_amount - match_split_amount) < 1e-6) - /* bug#347791: Double type shouldn't be compared for exact - equality, so we're using fabs() instead. */ - /*if (gnc_numeric_equal(xaccSplitGetAmount - (new_trans_fsplit), - xaccSplitGetAmount(split))) - -- gnc_numeric_equal is an expensive function call */ - { - prob = prob + 3; - /*DEBUG("heuristics: probability + 3 (amount)");*/ - } - else if (fabs (downloaded_split_amount - match_split_amount) <= - fuzzy_amount_difference) - { - /* ATM fees are sometimes added directly in the transaction. - So you withdraw 100$ and get charged 101,25$ in the same - transaction */ - prob = prob + 2; - /*DEBUG("heuristics: probability + 2 (amount)");*/ - } - else - { - /* If a transaction's amount doesn't match within the - threshold, it's very unlikely to be the same transaction - so we give it an extra -5 penalty */ - prob = prob - 5; - /* DEBUG("heuristics: probability - 1 (amount)"); */ - } - - /* Date heuristics */ - match_time = xaccTransGetDate (xaccSplitGetParent (split)); - download_time = xaccTransGetDate (new_trans); - datediff_day = llabs(match_time - download_time) / 86400; - /* Sorry, there are not really functions around at all that - provide for less hacky calculation of days of date - differences. Whatever. On the other hand, the difference - calculation itself will work regardless of month/year - turnarounds. */ - /*DEBUG("diff day %d", datediff_day);*/ - if (datediff_day == 0) - { - prob = prob + 3; - /*DEBUG("heuristics: probability + 3 (date)");*/ - } - else if (datediff_day <= date_threshold) - { - prob = prob + 2; - /*DEBUG("heuristics: probability + 2 (date)");*/ - } - else if (datediff_day > date_not_threshold) - { - /* Extra penalty if that split lies awfully far away from - the given one. */ - prob = prob - 5; - /*DEBUG("heuristics: probability - 5 (date)"); */ - /* Changed 2005-02-21: Revert the hard-limiting behaviour - back to the previous large penalty. (Changed 2004-11-27: - The penalty is so high that we can forget about this - split anyway and skip the rest of the tests.) */ - } - - /* Check if date and amount are identical */ - update_proposed = (prob < 6); - - /* Check number heuristics */ - { - const char *new_trans_str = gnc_get_num_action(new_trans, new_trans_fsplit); - if (new_trans_str && strlen(new_trans_str) != 0) - { - long new_trans_number, split_number; - const gchar *split_str; - char *endptr; - gboolean conversion_ok = TRUE; - - /* To distinguish success/failure after strtol call */ - errno = 0; - new_trans_number = strtol(new_trans_str, &endptr, 10); - /* Possible addressed problems: over/underflow, only non - numbers on string and string empty */ - if (errno || endptr == new_trans_str) - conversion_ok = FALSE; - - split_str = gnc_get_num_action (xaccSplitGetParent (split), split); - errno = 0; - split_number = strtol(split_str, &endptr, 10); - if (errno || endptr == split_str) - conversion_ok = FALSE; - - if ( (conversion_ok && (split_number == new_trans_number)) || - (g_strcmp0(new_trans_str, split_str) == 0) ) - { - /* An exact match of the Check number gives a +4 */ - prob += 4; - /*DEBUG("heuristics: probability + 4 (Check number)");*/ - } - else if (strlen(new_trans_str) > 0 && strlen(split_str) > 0) - { - /* If both number are not empty yet do not match, add a - little extra penalty */ - prob -= 2; - } - } - } - - /* Memo heuristics */ - { - const char *memo = xaccSplitGetMemo(new_trans_fsplit); - if (memo && strlen(memo) != 0) - { - if (safe_strcasecmp(memo, xaccSplitGetMemo(split)) == 0) - { - /* An exact match of memo gives a +2 */ - prob = prob + 2; - /* DEBUG("heuristics: probability + 2 (memo)"); */ - } - else if ((strncasecmp(memo, xaccSplitGetMemo(split), - strlen(xaccSplitGetMemo(split)) / 2) - == 0)) - { - /* Very primitive fuzzy match worth +1. This matches the - first 50% of the strings to skip annoying transaction - number some banks seem to include in the memo but someone - should write something more sophisticated */ - prob = prob + 1; - /*DEBUG("heuristics: probability + 1 (memo)"); */ - } - } - } - - /* Description heuristics */ - { - const char *descr = xaccTransGetDescription(new_trans); - if (descr && strlen(descr) != 0) - { - if (safe_strcasecmp(descr, - xaccTransGetDescription(xaccSplitGetParent(split))) - == 0) - { - /*An exact match of Description gives a +2 */ - prob = prob + 2; - /*DEBUG("heuristics: probability + 2 (description)");*/ - } - else if ((strncasecmp(descr, - xaccTransGetDescription (xaccSplitGetParent(split)), - strlen(xaccTransGetDescription (new_trans)) / 2) - == 0)) - { - /* Very primitive fuzzy match worth +1. This matches the - first 50% of the strings to skip annoying transaction - number some banks seem to include in the memo but someone - should write something more sophisticated */ - prob = prob + 1; - /*DEBUG("heuristics: probability + 1 (description)"); */ - } - } - } - - /* Is the probability high enough? Otherwise do nothing and return. */ - if (prob < display_threshold) - { - return; - } - - /* The probability is high enough, so allocate an object - here. Allocating it only when it's actually being used is - probably quite some performance gain. */ - match_info = g_new0(GNCImportMatchInfo, 1); - - match_info->probability = prob; - match_info->update_proposed = update_proposed; - match_info->split = split; - match_info->trans = xaccSplitGetParent(split); - - - /* Append that to the list. Do not use g_list_append because - it is slow. The list is sorted afterwards anyway. */ - trans_info->match_list = - g_list_prepend(trans_info->match_list, - match_info); - } -}/* end split_find_match */ - -/*********************************************************************** - */ - -/* append the imported transaction description to the matched transaction description */ -static void -desc_append (Transaction* selected_match_trans, gchar *new_desc) -{ - const gchar* curr_desc = xaccTransGetDescription (selected_match_trans); - gchar* tmp = g_strconcat(curr_desc, "|", new_desc, NULL); - xaccTransSetDescription (selected_match_trans, tmp); - g_free (tmp); -} - -/* append the imported transaction notes to the matched transaction notes */ -static void -notes_append (Transaction* selected_match_trans, gchar* new_notes) -{ - const gchar* curr_notes = xaccTransGetNotes (selected_match_trans); - gchar* tmp = g_strconcat (curr_notes, "|", new_notes, NULL); - xaccTransSetNotes (selected_match_trans, tmp ); - g_free (tmp); -} - -static char* -maybe_append_string (const char* match_string, const char* imp_string) -{ - char *norm_match_string, *norm_imp_string, *retval = NULL; - - if (!(match_string && *match_string)) - return g_strdup(imp_string); - - if (!(imp_string && *imp_string)) - return retval; - - norm_match_string = g_utf8_normalize (match_string, -1, G_NORMALIZE_NFC); - norm_imp_string = g_utf8_normalize (imp_string, -1, G_NORMALIZE_NFC); - - if (g_utf8_strlen (norm_imp_string, -1) > g_utf8_strlen (norm_match_string, -1) || - !strstr (norm_match_string, norm_imp_string)) - retval = g_strconcat(match_string, "|", imp_string, NULL); - - g_free (norm_match_string); - g_free (norm_imp_string); - return retval; - -} - -/* Append or replace transaction description and notes - * depending on the Append checkbox - */ -static void -update_desc_and_notes (const GNCImportTransInfo* trans_info) -{ - GNCImportMatchInfo* selected_match = - gnc_import_TransInfo_get_selected_match (trans_info); - Transaction* imp_trans = gnc_import_TransInfo_get_trans (trans_info); - Transaction* match_trans = selected_match->trans; - - if (trans_info->append_text) - { - gchar *repl_str; - - repl_str = - maybe_append_string (xaccTransGetDescription(match_trans), - xaccTransGetDescription(imp_trans)); - if (repl_str) - xaccTransSetDescription(match_trans, repl_str); - g_free (repl_str); - - repl_str = - maybe_append_string (xaccTransGetNotes(match_trans), - xaccTransGetNotes(imp_trans)); - if (repl_str) - xaccTransSetNotes (match_trans, repl_str); - g_free (repl_str); - } - else - { - // replace the matched transaction description with the imported transaction description - xaccTransSetDescription (selected_match->trans, - xaccTransGetDescription (imp_trans)); - // replace the matched transaction notes with the imported transaction notes - xaccTransSetNotes (selected_match->trans, - xaccTransGetNotes (imp_trans)); - } -} - -/** /brief -- Processes one match - according to its selected action. */ -gboolean -gnc_import_process_trans_item (GncImportMatchMap *matchmap, - GNCImportTransInfo *trans_info) -{ - Split * other_split; - gnc_numeric imbalance_value; - Transaction *trans; - - /* DEBUG("Begin"); */ - - g_assert (trans_info); - /*DEBUG("Iteration %d, action %d, split %s", i, - trans_info->action, - xaccTransGetDescription (gnc_import_TransInfo_get_trans - (trans_info)))*/ - switch (gnc_import_TransInfo_get_action (trans_info)) - { - case GNCImport_SKIP: - return FALSE; - case GNCImport_ADD: - /* Transaction gets imported. */ - - /* Is the transaction not balanced and there is a non-NULL destination account? */ - if (gnc_import_TransInfo_is_balanced(trans_info) == FALSE - && gnc_import_TransInfo_get_destacc(trans_info) != NULL) - { - /* Create the 'other' split. */ - Split *split = - xaccMallocSplit - (gnc_account_get_book - (gnc_import_TransInfo_get_destacc (trans_info))); - xaccTransAppendSplit - (gnc_import_TransInfo_get_trans (trans_info), split); - xaccAccountInsertSplit - (gnc_import_TransInfo_get_destacc (trans_info), split); - /*xaccSplitSetBaseValue - (split, - gnc_numeric_neg(xaccTransGetImbalance - (gnc_import_TransInfo_get_trans (trans_info))), - xaccTransGetCurrency - (gnc_import_TransInfo_get_trans (trans_info)));*/ - { - /* This is a quick workaround for the bug described in - http://lists.gnucash.org/pipermail/gnucash-devel/2003-August/009982.html - Assume that importers won't create transactions involving two or more - currencies so we can use xaccTransGetImbalanceValue. */ - imbalance_value = - gnc_numeric_neg (xaccTransGetImbalanceValue - (gnc_import_TransInfo_get_trans (trans_info))); - xaccSplitSetValue (split, imbalance_value); - xaccSplitSetAmount (split, imbalance_value); - } - /*xaccSplitSetMemo (split, _("Auto-Balance split")); - -- disabled due to popular request */ - } - - xaccSplitSetReconcile(gnc_import_TransInfo_get_fsplit (trans_info), CREC); - /*Set reconcile date to today*/ - xaccSplitSetDateReconciledSecs(gnc_import_TransInfo_get_fsplit (trans_info), - gnc_time (NULL)); - /* Done editing. */ - trans = gnc_import_TransInfo_get_trans (trans_info); - xaccTransCommitEdit(trans); - xaccTransRecordPrice(trans, PRICE_SOURCE_SPLIT_IMPORT); - return TRUE; - case GNCImport_UPDATE: - { - GNCImportMatchInfo *selected_match = - gnc_import_TransInfo_get_selected_match(trans_info); - - /* If there is no selection, ignore this transaction. */ - if (!selected_match) - { - PWARN("No matching translaction to be cleared was chosen. Imported transaction will be ignored."); - break; - } - - /* Transaction gets not imported but the matching one gets - updated and reconciled. */ - if (gnc_import_MatchInfo_get_split(selected_match) == NULL) - { - PERR("The split I am trying to update and reconcile is NULL, shouldn't happen!"); - } - else - { - /* Update and reconcile the matching transaction */ - /*DEBUG("BeginEdit selected_match")*/ - xaccTransBeginEdit(selected_match->trans); - - xaccTransSetDatePostedSecsNormalized(selected_match->trans, - xaccTransGetDate(xaccSplitGetParent( - gnc_import_TransInfo_get_fsplit(trans_info)))); - - xaccSplitSetAmount(selected_match->split, - xaccSplitGetAmount( - gnc_import_TransInfo_get_fsplit(trans_info))); - xaccSplitSetValue(selected_match->split, - xaccSplitGetValue( - gnc_import_TransInfo_get_fsplit(trans_info))); - - imbalance_value = xaccTransGetImbalanceValue( - gnc_import_TransInfo_get_trans(trans_info)); - other_split = xaccSplitGetOtherSplit(selected_match->split); - if (!gnc_numeric_zero_p(imbalance_value) && other_split) - { - if (xaccSplitGetReconcile(other_split) == NREC) - { - imbalance_value = gnc_numeric_neg(imbalance_value); - xaccSplitSetValue(other_split, imbalance_value); - xaccSplitSetAmount(other_split, imbalance_value); - } - /* else GC will automatically insert a split to equity - to balance the transaction */ - } - - update_desc_and_notes( trans_info); - - if (xaccSplitGetReconcile(selected_match->split) == NREC) - { - xaccSplitSetReconcile(selected_match->split, CREC); - } - - /* Set reconcile date to today */ - xaccSplitSetDateReconciledSecs(selected_match->split, gnc_time (NULL)); - - /* Copy the online id to the reconciled transaction, so - the match will be remembered */ - if (gnc_import_split_has_online_id(trans_info->first_split)) - { - char *online_id = gnc_import_get_split_online_id - (trans_info->first_split); - gnc_import_set_split_online_id(selected_match->split, online_id); - g_free (online_id); - } - - /* Done editing. */ - /*DEBUG("CommitEdit selected_match")*/ - xaccTransCommitEdit(selected_match->trans); - - /* Store the mapping to the other account in the MatchMap. */ - matchmap_store_destination(matchmap, trans_info, TRUE); - - /* Erase the downloaded transaction */ - xaccTransDestroy(trans_info->trans); - /*DEBUG("CommitEdit trans")*/ - xaccTransCommitEdit(trans_info->trans); - /* Very important: Make sure the freed transaction is not freed again! */ - trans_info->trans = NULL; - } - } - return TRUE; - case GNCImport_CLEAR: - { - GNCImportMatchInfo *selected_match = - gnc_import_TransInfo_get_selected_match (trans_info); - - /* If there is no selection, ignore this transaction. */ - if (!selected_match) - { - PWARN("No matching translaction to be cleared was chosen. Imported transaction will be ignored."); - break; - } - - /* Transaction gets not imported but the matching one gets - reconciled. */ - if (gnc_import_MatchInfo_get_split (selected_match) == NULL) - { - PERR("The split I am trying to reconcile is NULL, shouldn't happen!"); - } - else - { - /* Reconcile the matching transaction */ - /*DEBUG("BeginEdit selected_match")*/ - xaccTransBeginEdit(selected_match->trans); - - if (xaccSplitGetReconcile - (selected_match->split) == NREC) - xaccSplitSetReconcile - (selected_match->split, CREC); - /* Set reconcile date to today */ - xaccSplitSetDateReconciledSecs - (selected_match->split, gnc_time (NULL)); - - /* Copy the online id to the reconciled transaction, so - the match will be remembered */ - if (gnc_import_split_has_online_id(trans_info->first_split)) - { - char *online_id = gnc_import_get_split_online_id - (trans_info->first_split); - gnc_import_set_split_online_id (selected_match->split, online_id); - g_free (online_id); - } - - /* Done editing. */ - /*DEBUG("CommitEdit selected_match")*/ - xaccTransCommitEdit - (selected_match->trans); - - /* Store the mapping to the other account in the MatchMap. */ - matchmap_store_destination (matchmap, trans_info, TRUE); - - /* Erase the downloaded transaction */ - xaccTransDestroy(trans_info->trans); - /*DEBUG("CommitEdit trans")*/ - xaccTransCommitEdit(trans_info->trans); - /* Very important: Make sure the freed transaction is not freed again! */ - trans_info->trans = NULL; - } - } - return TRUE; - default: - DEBUG("Invalid GNCImportAction for this imported transaction."); - break; - } - /*DEBUG("End");*/ - return FALSE; -} - -/********************************************************************\ - * check_trans_online_id() Callback function used by - * gnc_import_exists_online_id. Takes pointers to transaction and split, - * returns 0 if their online_ids do NOT match, or if the split - * belongs to the transaction -\********************************************************************/ -static gint check_trans_online_id(Transaction *trans1, void *user_data) -{ - Account *account; - Split *split1; - Split *split2 = user_data; - gchar *online_id1, *online_id2; - gint retval; - - account = xaccSplitGetAccount(split2); - split1 = xaccTransFindSplitByAccount(trans1, account); - if (split1 == split2) - return 0; - - /* hack - we really want to iterate over the _splits_ of the account - instead of the transactions */ - g_assert(split1 != NULL); - - online_id1 = gnc_import_get_split_online_id (split1); - - if (!online_id1 || !online_id1[0]) - { - if (online_id1) - g_free (online_id1); - online_id1 = gnc_import_get_trans_online_id (trans1); - } - - online_id2 = gnc_import_get_split_online_id(split2); - - retval = (!online_id1 || !online_id2 || strcmp (online_id1, online_id2)) ? 0 : 1; - - g_free (online_id1); - g_free (online_id2); - return retval; -} - -static GHashTable* -hash_account_online_ids (Account *account) -{ - GHashTable* acct_hash = g_hash_table_new_full - (g_str_hash, g_str_equal, g_free, NULL); - for (GList *n = xaccAccountGetSplitList (account) ; n; n = n->next) - { - if (gnc_import_split_has_online_id (n->data)) - { - char *id = gnc_import_get_split_online_id (n->data); - g_hash_table_insert (acct_hash, (void*) id, GINT_TO_POINTER (1)); - } - } - return acct_hash; -} - -/** Checks whether the given transaction's online_id already exists in - its parent account. */ -gboolean gnc_import_exists_online_id (Transaction *trans, GHashTable* acct_id_hash) -{ - gboolean online_id_exists = FALSE; - Account *dest_acct; - Split *source_split; - char *source_online_id; - - /* Look for an online_id in the first split */ - source_split = xaccTransGetSplit(trans, 0); - g_assert(source_split); - - source_online_id = gnc_import_get_split_online_id (source_split); - - // No online id, no point in continuing. We'd crash if we tried. - if (!source_online_id) - return FALSE; - - // Create a hash per account of a hash of all split IDs. Then the - // test below will be fast if we have many transactions to import. - dest_acct = xaccSplitGetAccount (source_split); - if (!g_hash_table_contains (acct_id_hash, dest_acct)) - g_hash_table_insert (acct_id_hash, dest_acct, - hash_account_online_ids (dest_acct)); - online_id_exists = g_hash_table_contains (g_hash_table_lookup (acct_id_hash, dest_acct), - source_online_id); - - /* If it does, abort the process for this transaction, since it is - already in the system. */ - if (online_id_exists == TRUE) - { - DEBUG("%s", "Transaction with same online ID exists, destroying current transaction"); - xaccTransDestroy(trans); - xaccTransCommitEdit(trans); - } - g_free (source_online_id); - return online_id_exists; -} - - -/* ****************************************************************** - */ - -/** Create a new object of GNCImportTransInfo here. */ -GNCImportTransInfo * -gnc_import_TransInfo_new (Transaction *trans, GncImportMatchMap *matchmap) -{ - GNCImportTransInfo *transaction_info; - Split *split; - g_assert (trans); - - transaction_info = g_new0(GNCImportTransInfo, 1); - - transaction_info->trans = trans; - /* Only use first split, the source split */ - split = xaccTransGetSplit(trans, 0); - g_assert(split); - transaction_info->first_split = split; - - /* Try to find a previously selected destination account - string match for the ADD action */ - gnc_import_TransInfo_set_destacc (transaction_info, - matchmap_find_destination (matchmap, transaction_info), - FALSE); - return transaction_info; -} - - -/** compare_probability() is used by g_list_sort to sort by probability */ -static gint compare_probability (gconstpointer a, - gconstpointer b) -{ - return(((GNCImportMatchInfo *)b)->probability - - ((GNCImportMatchInfo *)a)->probability); -} - -/** Iterates through all splits of the originating account of - * trans_info. Sorts the resulting list and sets the selected_match - * and action fields in the trans_info. - */ -void -gnc_import_TransInfo_init_matches (GNCImportTransInfo *trans_info, - GNCImportSettings *settings) -{ - GNCImportMatchInfo * best_match = NULL; - g_assert (trans_info); - - if (trans_info->match_list != NULL) - { - trans_info->match_list = g_list_sort(trans_info->match_list, - compare_probability); - best_match = g_list_nth_data(trans_info->match_list, 0); - gnc_import_TransInfo_set_selected_match_info (trans_info, - best_match, - FALSE); - if (best_match != NULL && - best_match->probability >= gnc_import_Settings_get_clear_threshold(settings)) - { - trans_info->action = GNCImport_CLEAR; - } - else if (best_match == NULL || - best_match->probability <= gnc_import_Settings_get_add_threshold(settings)) - { - trans_info->action = GNCImport_ADD; - } - else if (gnc_import_Settings_get_action_skip_enabled(settings)) - { - trans_info->action = GNCImport_SKIP; - } - else if (gnc_import_Settings_get_action_update_enabled(settings)) - { - trans_info->action = GNCImport_UPDATE; - } - else - { - trans_info->action = GNCImport_ADD; - } - } - else - { - trans_info->action = GNCImport_ADD; - } - if (best_match && - trans_info->action == GNCImport_CLEAR && - gnc_import_Settings_get_action_update_enabled(settings)) - { - if (best_match->update_proposed) - { - trans_info->action = GNCImport_UPDATE; - } - } - - trans_info->previous_action = trans_info->action; -} - - -/* Try to automatch a transaction to a destination account if the */ -/* transaction hasn't already been manually assigned to another account */ -gboolean -gnc_import_TransInfo_refresh_destacc (GNCImportTransInfo *transaction_info, - GncImportMatchMap *matchmap) -{ - Account *orig_destacc; - Account *new_destacc = NULL; - g_assert(transaction_info); - - orig_destacc = gnc_import_TransInfo_get_destacc(transaction_info); - - /* if we haven't manually selected a destination account for this transaction */ - if (gnc_import_TransInfo_get_destacc_selected_manually(transaction_info) == FALSE) - { - /* Try to find the destination account for this transaction based on prior ones */ - new_destacc = matchmap_find_destination(matchmap, transaction_info); - gnc_import_TransInfo_set_destacc(transaction_info, new_destacc, FALSE); - } - else - { - new_destacc = orig_destacc; - } - - /* account has changed */ - if (new_destacc != orig_destacc) - { - return TRUE; - } - else /* account is the same */ - { - return FALSE; - } -} - - -/** @} */ diff --git a/gnucash/import-export/import-backend.cpp b/gnucash/import-export/import-backend.cpp new file mode 100644 index 0000000000..a372a20edd --- /dev/null +++ b/gnucash/import-export/import-backend.cpp @@ -0,0 +1,1189 @@ +/********************************************************************\ + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License* + * along with this program; if not, contact: * + * * + * Free Software Foundation Voice: +1-617-542-5942 * + * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 * + * Boston, MA 02110-1301, USA gnu@gnu.org * +\********************************************************************/ +/** @addtogroup Import_Export + @{ */ +/** @internal + @file import-backend.c + @brief import-backend.c: Generic importer backend implementation (duplicate matching algorithm, action handling, etc.) + @author Copyright (C) 2002 Benoit GrĂ©goire + @author Christian Stimming + @author Copyright (c) 2006 David Hampton +*/ + +#include + +#include +#include +#include +#include + +#include + +extern "C" { +#include "import-backend.h" +#include "import-utilities.h" +#include "Account.h" +#include "Query.h" +#include "gnc-engine.h" +#include "engine-helpers.h" +#include "gnc-prefs.h" +#include "gnc-ui-util.h" +} + +#include + +#define GNCIMPORT_DESC "desc" +#define GNCIMPORT_MEMO "memo" +#define GNCIMPORT_PAYEE "payee" + +/********************************************************************\ + * Constants * +\********************************************************************/ + +static QofLogModule log_module = GNC_MOD_IMPORT; + +/********************************************************************\ + * Forward declared prototypes * +\********************************************************************/ + +static void matchmap_store_destination(Account* base_acc, + GNCImportTransInfo* trans_info, + gboolean use_match); + +static void trans_info_calculate_dest_amount (GNCImportTransInfo *info); + + +/********************************************************************\ + * Structures passed between the functions * +\********************************************************************/ + +struct _selected_match_info +{ + GNCImportMatchInfo *selected_match; + gboolean selected_manually; +}; + +struct _transactioninfo +{ + Transaction * trans; + Split * first_split; + + /* GList of GNCImportMatchInfo's, one for each possible duplicate match. */ + GList * match_list; + GNCImportSelectedMatchInfo selected_match_info; + + GNCImportAction action; + GNCImportAction previous_action; + + /* A list of tokenized strings to use for bayesian matching purposes */ + GList * match_tokens; + + /* In case of a single destination account it is stored here. */ + Account *dest_acc; + gboolean dest_acc_selected_manually; + + /* Reference id to link gnc transaction to external object. E.g. aqbanking job id. */ + guint32 ref_id; + + /* When updating a matched transaction, append Description and Notes instead of replacing */ + gboolean append_text; + + /* Extra data we can use to build the balancing split. It may passed on by the + * code that calls the generic importer */ + gnc_numeric lsplit_price; + char *lsplit_action; + char *lsplit_memo; + char lsplit_rec_state; + time64 lsplit_rec_date; + + gnc_numeric lsplit_value; + /* Amount for the balancing split. This can only be calculated when + * the destination account is known and may require an exchange rate + * if that account is not in the same commodity as the transaction. */ + gnc_numeric lsplit_amount; +}; + +/* Some simple getters and setters for the above data types. */ + +GList * +gnc_import_TransInfo_get_match_list (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->match_list; +} + +void +gnc_import_TransInfo_set_match_list (GNCImportTransInfo *info, GList* match_list) +{ + g_assert (info); + info->match_list = match_list; + if (match_list) + info->selected_match_info.selected_match = static_cast(match_list->data); + else + { + info->selected_match_info.selected_match = nullptr; + gnc_import_TransInfo_set_action (info, GNCImport_ADD); + } +} + +Transaction * +gnc_import_TransInfo_get_trans (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->trans; +} + +gboolean +gnc_import_TransInfo_is_balanced (const GNCImportTransInfo *info) +{ + g_assert (info); + return gnc_numeric_zero_p(xaccTransGetImbalanceValue(gnc_import_TransInfo_get_trans(info))); +} + +Split * +gnc_import_TransInfo_get_fsplit (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->first_split; +} + +GNCImportMatchInfo * +gnc_import_TransInfo_get_selected_match (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->selected_match_info.selected_match; +} + +void +gnc_import_TransInfo_set_selected_match_info (GNCImportTransInfo *info, + GNCImportMatchInfo *match, + gboolean selected_manually) +{ + g_assert (info); + info->selected_match_info.selected_match = match; + info->selected_match_info.selected_manually = selected_manually; +} + +gboolean +gnc_import_TransInfo_get_match_selected_manually (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->selected_match_info.selected_manually; +} + +GNCImportAction +gnc_import_TransInfo_get_action (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->action; +} + +void +gnc_import_TransInfo_set_action (GNCImportTransInfo *info, + GNCImportAction action) +{ + g_assert (info); + if (action != info->action) + { + info->previous_action = info->action; + info->action = action; + } +} + +Account * +gnc_import_TransInfo_get_destacc (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->dest_acc; +} +void gnc_import_TransInfo_set_destacc (GNCImportTransInfo *info, + Account *acc, + gboolean selected_manually) +{ + g_assert (info); + info->dest_acc = acc; + info->dest_acc_selected_manually = selected_manually; + + /* Store the mapping to the other account in the MatchMap. */ + if (selected_manually) + matchmap_store_destination (nullptr, info, false); + + trans_info_calculate_dest_amount (info); +} + +gboolean +gnc_import_TransInfo_get_destacc_selected_manually (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->dest_acc_selected_manually; +} + +guint32 +gnc_import_TransInfo_get_ref_id (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->ref_id; +} + +void +gnc_import_TransInfo_set_ref_id (GNCImportTransInfo *info, + guint32 ref_id) +{ + g_assert (info); + info->ref_id = ref_id; +} + +gnc_numeric +gnc_import_TransInfo_get_price (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->lsplit_price; +} + +void +gnc_import_TransInfo_set_price (GNCImportTransInfo *info, + gnc_numeric lprice) +{ + g_assert (info); + info->lsplit_price = lprice; +} + +gnc_numeric +gnc_import_TransInfo_get_dest_amount (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->lsplit_amount; +} + +gnc_numeric +gnc_import_TransInfo_get_dest_value (const GNCImportTransInfo *info) +{ + g_assert (info); + return info->lsplit_value; +} + +void +gnc_import_TransInfo_set_last_split_info (GNCImportTransInfo *info, + GNCImportLastSplitInfo *lsplit) +{ + g_assert (info); + if (lsplit) + { + info->lsplit_price = lsplit->price; + info->lsplit_action = g_strdup(lsplit->action); + info->lsplit_memo = g_strdup(lsplit->memo); + info->lsplit_rec_state = lsplit->rec_state; + info->lsplit_rec_date = lsplit->rec_date; + } +} + +void +gnc_import_TransInfo_set_append_text (GNCImportTransInfo *info, + gboolean append_text) +{ + g_assert (info); + info->append_text = append_text; +} + + +Split * +gnc_import_MatchInfo_get_split (const GNCImportMatchInfo * info) +{ + g_assert (info); + return info->split; +} + +gint +gnc_import_MatchInfo_get_probability (const GNCImportMatchInfo * info) +{ + if (info) + return info->probability; + else + return 0; +} + +void gnc_import_TransInfo_delete (GNCImportTransInfo *info) +{ + if (info) + { + g_list_free (info->match_list); + /*If the transaction exists and is still open, it must be destroyed*/ + if (xaccTransIsOpen(info->trans)) + { + xaccTransDestroy(info->trans); + xaccTransCommitEdit(info->trans); + } + g_list_free_full (info->match_tokens, g_free); + g_free(info->lsplit_action); + g_free(info->lsplit_memo); + + g_free(info); + } +} + +GdkPixbuf* gen_probability_pixbuf(gint score_original, GNCImportSettings *settings, GtkWidget * widget) +{ + constexpr gint height = 15; + constexpr gint width_each_bar = 7; + constexpr gint width_first_bar = 1; + constexpr gint num_colors = 5; + gchar * xpm[2 + num_colors + height]; + + g_assert(settings); + g_assert(widget); + + auto score = std::max (0, score_original); + + /* Add size definition to xpm */ + xpm[0] = g_strdup_printf("%d%s%d%s%d%s", (width_each_bar * score) + width_first_bar/*width*/, " ", height, " ", num_colors, " 1"/*characters per pixel*/); + + /* Define used colors */ + xpm[1] = g_strdup(" c None"); + xpm[2] = g_strdup("g c green"); + xpm[3] = g_strdup("y c yellow"); + xpm[4] = g_strdup("r c red"); + xpm[5] = g_strdup("b c black"); + + auto add_threshold = gnc_import_Settings_get_add_threshold(settings); + auto clear_threshold = gnc_import_Settings_get_clear_threshold(settings); + for (int i = 0; i < height; i++) + { + xpm[num_colors+1+i] = g_new0(char, (width_each_bar * score) + width_first_bar + 1); + for (int j = 0; j <= score; j++) + { + if (j == 0) + strcat(xpm[num_colors+1+i], "b"); + else if (i == 0 || i == height - 1) + strcat(xpm[num_colors+1+i], "bbbbbb "); + else if (j <= add_threshold) + strcat(xpm[num_colors+1+i], "brrrrb "); + else if (j >= clear_threshold) + strcat(xpm[num_colors+1+i], "bggggb "); + else + strcat(xpm[num_colors+1+i], "byyyyb "); + } + } + + auto retval = gdk_pixbuf_new_from_xpm_data((const gchar **)xpm); + for (int i = 0; i <= num_colors + height; i++) + g_free(xpm[i]); + + return retval; +} + +/************************************************************************* + * MatchMap related functions (storing and retrieving) + */ + +/* Tokenize a string and append the tokens to an existing GList + * (or an empty GList) + */ +static GList* +tokenize_string(GList* existing_tokens, const char *string) +{ + char **tokenized_strings = g_strsplit(string, " ", 0); + char **stringpos = tokenized_strings; + + /* add each unique non-empty token to the token GList */ + while (stringpos && *stringpos) + { + if ((strlen(*stringpos) > 0) && + (!g_list_find_custom (existing_tokens, *stringpos, (GCompareFunc)g_strcmp0))) + existing_tokens = g_list_prepend(existing_tokens, g_strdup(*stringpos)); + + stringpos++; + } + + /* free up the strings that g_strsplit() created */ + g_strfreev(tokenized_strings); + + return existing_tokens; +} + +/* create and return a list of tokens for a given transaction info. */ +static GList* +TransactionGetTokens(GNCImportTransInfo *info) +{ + + g_return_val_if_fail (info, nullptr); + if (info->match_tokens) return info->match_tokens; + + auto transaction = gnc_import_TransInfo_get_trans(info); + g_assert(transaction); + + /* make tokens from the transaction description */ + auto text = xaccTransGetDescription(transaction); + GList *tokens = nullptr; + tokens = tokenize_string(tokens, text); + + /* The day of week the transaction occurred is a good indicator of + * what account this transaction belongs in. Get the date and convert + * it to day of week as a token + */ + auto transtime = xaccTransGetDate(transaction); + auto tm_struct = gnc_gmtime(&transtime); + char local_day_of_week[16]; + if (!qof_strftime(local_day_of_week, sizeof(local_day_of_week), "%A", tm_struct)) + PERR("TransactionGetTokens: error, strftime failed\n"); + gnc_tm_free (tm_struct); + /* we cannot add a locally allocated string to this array, dup it so + * it frees the same way the rest do + */ + tokens = g_list_prepend(tokens, g_strdup(local_day_of_week)); + + /* make tokens from the memo of each split of this transaction */ + for (GList *node=xaccTransGetSplitList (transaction); node; node=node->next) + { + text = xaccSplitGetMemo(static_cast(node->data)); + tokens = tokenize_string(tokens, text); + } + + info->match_tokens = tokens; + return tokens; +} + +/* searches using the GNCImportTransInfo through all existing transactions + * if there is an exact match of the description and memo + */ +static Account * +matchmap_find_destination (Account *base_acc, GNCImportTransInfo *info) +{ + g_assert (info); + auto orig_acc = (base_acc ? base_acc : xaccSplitGetAccount + (gnc_import_TransInfo_get_fsplit (info))); + + Account *result = nullptr; + if (gnc_prefs_get_bool (GNC_PREFS_GROUP_IMPORT, GNC_PREF_USE_BAYES)) + { + /* get the tokens for this transaction* */ + GList* tokens = TransactionGetTokens(info); + + /* try to find the destination account for this transaction from its tokens */ + result = gnc_account_imap_find_account_bayes(orig_acc, tokens); + + } + else + result = gnc_account_imap_find_account + (orig_acc, GNCIMPORT_DESC, + xaccTransGetDescription (gnc_import_TransInfo_get_trans (info))); + + return result; +} + +/** Store the destination account from trans_info in the matchmap. If + 'use_match' is true, the destination account of the selected + matching/duplicate transaction is used; otherwise, the stored + destination_acc pointer is used. */ +static void +matchmap_store_destination (Account *base_acc, + GNCImportTransInfo *trans_info, + gboolean use_match) +{ + g_assert (trans_info); + + /* This will store the destination account of the selected match if + the reconcile match selected has only two splits. */ + Account *dest = nullptr; + if (use_match) + dest = xaccSplitGetAccount + (xaccSplitGetOtherSplit + (gnc_import_MatchInfo_get_split + (gnc_import_TransInfo_get_selected_match (trans_info)))); + else + dest = gnc_import_TransInfo_get_destacc (trans_info); + if (!dest) + return; + + auto orig_acc = (base_acc ? base_acc : xaccSplitGetAccount + (gnc_import_TransInfo_get_fsplit (trans_info))); + + if (gnc_prefs_get_bool (GNC_PREFS_GROUP_IMPORT, GNC_PREF_USE_BAYES)) + { + /* tokenize this transaction */ + auto tokens = TransactionGetTokens(trans_info); + + /* add the tokens to the imap with the given destination account */ + gnc_account_imap_add_account_bayes(orig_acc, tokens, dest); + } + else + { + auto desc = xaccTransGetDescription + (gnc_import_TransInfo_get_trans (trans_info)); + auto memo = xaccSplitGetMemo + (gnc_import_TransInfo_get_fsplit (trans_info)); + + if (desc && *desc) + gnc_account_imap_add_account (orig_acc, GNCIMPORT_DESC, desc, dest); + if (memo && *memo) + gnc_account_imap_add_account (orig_acc, GNCIMPORT_MEMO, memo, dest); + } +} + + + +/** @brief The transaction matching heuristics are here. + */ +void split_find_match (GNCImportTransInfo * trans_info, + Split * split, + gint display_threshold, + gint date_threshold, + gint date_not_threshold, + double fuzzy_amount_difference) +{ + gint prob = 0; + + auto new_trans = gnc_import_TransInfo_get_trans (trans_info); + auto new_trans_fsplit = gnc_import_TransInfo_get_fsplit (trans_info); + + /* Matching heuristics */ + + /* Amount heuristics */ + auto downloaded_split_amount = + gnc_numeric_to_double (xaccSplitGetAmount(new_trans_fsplit)); + /*DEBUG(" downloaded_split_amount=%f", downloaded_split_amount);*/ + auto match_split_amount = gnc_numeric_to_double(xaccSplitGetAmount(split)); + /*DEBUG(" match_split_amount=%f", match_split_amount);*/ + if (fabs(downloaded_split_amount - match_split_amount) < 1e-6) + /* bug#347791: Double type shouldn't be compared for exact + equality, so we're using fabs() instead. */ + /*if (gnc_numeric_equal(xaccSplitGetAmount + (new_trans_fsplit), + xaccSplitGetAmount(split))) + -- gnc_numeric_equal is an expensive function call */ + { + prob = prob + 3; + /*DEBUG("heuristics: probability + 3 (amount)");*/ + } + else if (fabs (downloaded_split_amount - match_split_amount) <= + fuzzy_amount_difference) + { + /* ATM fees are sometimes added directly in the transaction. + So you withdraw 100$ and get charged 101,25$ in the same + transaction */ + prob = prob + 2; + /*DEBUG("heuristics: probability + 2 (amount)");*/ + } + else + { + /* If a transaction's amount doesn't match within the + threshold, it's very unlikely to be the same transaction + so we give it an extra -5 penalty */ + prob = prob - 5; + /* DEBUG("heuristics: probability - 1 (amount)"); */ + } + + /* Date heuristics */ + auto match_time = xaccTransGetDate (xaccSplitGetParent (split)); + auto download_time = xaccTransGetDate (new_trans); + auto datediff_day = llabs(match_time - download_time) / 86400; + /* Sorry, there are not really functions around at all that + provide for less hacky calculation of days of date + differences. Whatever. On the other hand, the difference + calculation itself will work regardless of month/year + turnarounds. */ + /*DEBUG("diff day %d", datediff_day);*/ + if (datediff_day == 0) + { + prob = prob + 3; + /*DEBUG("heuristics: probability + 3 (date)");*/ + } + else if (datediff_day <= date_threshold) + { + prob = prob + 2; + /*DEBUG("heuristics: probability + 2 (date)");*/ + } + else if (datediff_day > date_not_threshold) + { + /* Extra penalty if that split lies awfully far away from + the given one. */ + prob = prob - 5; + /*DEBUG("heuristics: probability - 5 (date)"); */ + /* Changed 2005-02-21: Revert the hard-limiting behaviour + back to the previous large penalty. (Changed 2004-11-27: + The penalty is so high that we can forget about this + split anyway and skip the rest of the tests.) */ + } + + /* Check if date and amount are identical */ + auto update_proposed = (prob < 6); + + /* Check number heuristics */ + auto new_trans_str = gnc_get_num_action(new_trans, new_trans_fsplit); + if (new_trans_str && *new_trans_str) + { + char *endptr; + auto conversion_ok = true; + + /* To distinguish success/failure after strtol call */ + errno = 0; + auto new_trans_number = strtol(new_trans_str, &endptr, 10); + /* Possible addressed problems: over/underflow, only non + numbers on string and string empty */ + conversion_ok = !(errno || endptr == new_trans_str); + + auto split_str = gnc_get_num_action (xaccSplitGetParent (split), split); + errno = 0; + auto split_number = strtol(split_str, &endptr, 10); + conversion_ok = !(errno || endptr == split_str); + + if ( (conversion_ok && (split_number == new_trans_number)) || + (g_strcmp0(new_trans_str, split_str) == 0) ) + { + /* An exact match of the Check number gives a +4 */ + prob += 4; + /*DEBUG("heuristics: probability + 4 (Check number)");*/ + } + else if (strlen(new_trans_str) > 0 && strlen(split_str) > 0) + { + /* If both number are not empty yet do not match, add a + little extra penalty */ + prob -= 2; + } + } + + /* Memo heuristics */ + auto memo = xaccSplitGetMemo(new_trans_fsplit); + if (memo && *memo) + { + if (safe_strcasecmp(memo, xaccSplitGetMemo(split)) == 0) + { + /* An exact match of memo gives a +2 */ + prob = prob + 2; + /* DEBUG("heuristics: probability + 2 (memo)"); */ + } + else if ((strncasecmp(memo, xaccSplitGetMemo(split), + strlen(xaccSplitGetMemo(split)) / 2) == 0)) + { + /* Very primitive fuzzy match worth +1. This matches the + first 50% of the strings to skip annoying transaction + number some banks seem to include in the memo but someone + should write something more sophisticated */ + prob = prob + 1; + /*DEBUG("heuristics: probability + 1 (memo)"); */ + } + } + + /* Description heuristics */ + auto descr = xaccTransGetDescription(new_trans); + if (descr && *descr) + { + if (safe_strcasecmp(descr, + xaccTransGetDescription(xaccSplitGetParent(split))) == 0) + { + /*An exact match of Description gives a +2 */ + prob = prob + 2; + /*DEBUG("heuristics: probability + 2 (description)");*/ + } + else if ((strncasecmp(descr, + xaccTransGetDescription (xaccSplitGetParent(split)), + strlen(xaccTransGetDescription (new_trans)) / 2) == 0)) + { + /* Very primitive fuzzy match worth +1. This matches the + first 50% of the strings to skip annoying transaction + number some banks seem to include in the description but someone + should write something more sophisticated */ + prob = prob + 1; + /*DEBUG("heuristics: probability + 1 (description)"); */ + } + } + + /* Is the probability high enough? Otherwise do nothing and return. */ + if (prob < display_threshold) + return; + + /* The probability is high enough, so allocate an object + here. Allocating it only when it's actually being used is + probably quite some performance gain. */ + auto match_info = g_new0(GNCImportMatchInfo, 1); + + match_info->probability = prob; + match_info->update_proposed = update_proposed; + match_info->split = split; + match_info->trans = xaccSplitGetParent(split); + + + /* Append that to the list. Do not use g_list_append because + it is slow. The list is sorted afterwards anyway. */ + trans_info->match_list = g_list_prepend(trans_info->match_list, match_info); +} + +/*********************************************************************** + */ + +/* append the imported transaction description to the matched transaction description */ +static void +desc_append (Transaction* selected_match_trans, gchar *new_desc) +{ + auto curr_desc = xaccTransGetDescription (selected_match_trans); + auto tmp = g_strconcat(curr_desc, "|", new_desc, nullptr); + xaccTransSetDescription (selected_match_trans, tmp); + g_free (tmp); +} + +/* append the imported transaction notes to the matched transaction notes */ +static void +notes_append (Transaction* selected_match_trans, gchar* new_notes) +{ + auto curr_notes = xaccTransGetNotes (selected_match_trans); + auto tmp = g_strconcat (curr_notes, "|", new_notes, nullptr); + xaccTransSetNotes (selected_match_trans, tmp ); + g_free (tmp); +} + +static char* +maybe_append_string (const char* match_string, const char* imp_string) +{ + if (!(match_string && *match_string)) + return g_strdup(imp_string); + + if (!(imp_string && *imp_string)) + return nullptr; + + auto norm_match_string = g_utf8_normalize (match_string, -1, G_NORMALIZE_NFC); + auto norm_imp_string = g_utf8_normalize (imp_string, -1, G_NORMALIZE_NFC); + + char *retval = nullptr; + if (g_utf8_strlen (norm_imp_string, -1) > g_utf8_strlen (norm_match_string, -1) || + !strstr (norm_match_string, norm_imp_string)) + retval = g_strconcat(match_string, "|", imp_string, nullptr); + + g_free (norm_match_string); + g_free (norm_imp_string); + return retval; + +} + +/* Append or replace transaction description and notes + * depending on the Append checkbox + */ +static void +update_desc_and_notes (const GNCImportTransInfo* trans_info) +{ + auto selected_match = gnc_import_TransInfo_get_selected_match (trans_info); + auto imp_trans = gnc_import_TransInfo_get_trans (trans_info); + + if (trans_info->append_text) + { + auto match_trans = selected_match->trans; + auto repl_str = + maybe_append_string (xaccTransGetDescription(match_trans), + xaccTransGetDescription(imp_trans)); + if (repl_str) + xaccTransSetDescription(match_trans, repl_str); + g_free (repl_str); + + repl_str = + maybe_append_string (xaccTransGetNotes(match_trans), + xaccTransGetNotes(imp_trans)); + if (repl_str) + xaccTransSetNotes (match_trans, repl_str); + g_free (repl_str); + } + else + { + xaccTransSetDescription (selected_match->trans, + xaccTransGetDescription (imp_trans)); + xaccTransSetNotes (selected_match->trans, + xaccTransGetNotes (imp_trans)); + } +} + +static void +process_reconcile(Account *base_acc, + GNCImportTransInfo *trans_info, + GNCImportMatchInfo *selected_match) +{ + /* Reconcile the matching transaction */ + /*DEBUG("BeginEdit selected_match")*/ + xaccTransBeginEdit(selected_match->trans); + + if (xaccSplitGetReconcile(selected_match->split) == NREC) + xaccSplitSetReconcile(selected_match->split, CREC); + + /* Set reconcile date to today */ + xaccSplitSetDateReconciledSecs(selected_match->split, gnc_time (nullptr)); + + /* Copy the online id to the reconciled transaction, so + * the match will be remembered */ + auto online_id = gnc_import_get_split_online_id(trans_info->first_split); + if (online_id && *online_id) + gnc_import_set_split_online_id(selected_match->split, online_id); + + g_free (online_id); + + /* Done editing. */ + /*DEBUG("CommitEdit selected_match")*/ + xaccTransCommitEdit(selected_match->trans); + + /* Store the mapping to the other account in the MatchMap. */ + matchmap_store_destination(base_acc, trans_info, true); + + /* Erase the downloaded transaction */ + xaccTransDestroy(trans_info->trans); + /*DEBUG("CommitEdit trans")*/ + xaccTransCommitEdit(trans_info->trans); + /* Very important: Make sure the freed transaction is not freed again! */ + trans_info->trans = nullptr; +} + +/** /brief -- Processes one match + according to its selected action. */ +gboolean +gnc_import_process_trans_item (Account *base_acc, + GNCImportTransInfo *trans_info) +{ + g_assert (trans_info); + /*DEBUG("Iteration %d, action %d, split %s", i, + trans_info->action, + xaccTransGetDescription (gnc_import_TransInfo_get_trans + (trans_info)))*/ + switch (gnc_import_TransInfo_get_action (trans_info)) + { + case GNCImport_SKIP: + return false; + case GNCImport_ADD: + /* Transaction gets imported. */ + if (!gnc_import_TransInfo_is_balanced(trans_info) + && gnc_import_TransInfo_get_destacc(trans_info)) + { + /* Create the 'other' split. */ + auto trans = gnc_import_TransInfo_get_trans (trans_info); + auto acct = gnc_import_TransInfo_get_destacc (trans_info); + auto split = xaccMallocSplit (gnc_account_get_book (acct)); + xaccTransAppendSplit (trans, split); + xaccAccountInsertSplit (acct, split); + auto imbalance_value = gnc_numeric_neg (xaccTransGetImbalanceValue (trans)); + xaccSplitSetValue (split, trans_info->lsplit_value); + if (!gnc_numeric_zero_p (trans_info->lsplit_amount)) + xaccSplitSetAmount (split, trans_info->lsplit_amount); + else + { + /* Bad! user asked to create a balancing split in an account with + * different currency/commodit than the transaction but didn't provide + * an exchange rate. + * Continue anyway pretenting split is in transaction currency. */ + xaccSplitSetAmount (split, trans_info->lsplit_value); + PWARN("Missing exchange rate while adding transaction '%s', will assume rate of 1", + xaccTransGetDescription (gnc_import_TransInfo_get_trans (trans_info))); + } + } + + xaccSplitSetReconcile(gnc_import_TransInfo_get_fsplit (trans_info), CREC); + /*Set reconcile date to today*/ + xaccSplitSetDateReconciledSecs(gnc_import_TransInfo_get_fsplit (trans_info), + gnc_time (nullptr)); + /* Done editing. */ + xaccTransCommitEdit(trans_info->trans); + xaccTransRecordPrice(trans_info->trans, PRICE_SOURCE_SPLIT_IMPORT); + return true; + case GNCImport_UPDATE: + { + auto selected_match = gnc_import_TransInfo_get_selected_match(trans_info); + + /* If there is no selection, ignore this transaction. */ + if (!selected_match) + { + PWARN("No matching transaction to be cleared was chosen. Imported transaction will be ignored."); + break; + } + + /* Transaction gets not imported but the matching one gets + updated and reconciled. */ + if (!gnc_import_MatchInfo_get_split(selected_match)) + PERR("The split I am trying to update and reconcile is nullptr, shouldn't happen!"); + else + { + /* Update and reconcile the matching transaction */ + /*DEBUG("BeginEdit selected_match")*/ + xaccTransBeginEdit(selected_match->trans); + + auto fsplit = gnc_import_TransInfo_get_fsplit(trans_info); + xaccTransSetDatePostedSecsNormalized(selected_match->trans, + xaccTransGetDate(xaccSplitGetParent(fsplit))); + + auto match_split_amount = xaccSplitGetAmount(selected_match->split); + xaccSplitSetAmount(selected_match->split, xaccSplitGetAmount(fsplit)); + xaccSplitSetValue(selected_match->split, xaccSplitGetValue(fsplit)); + + auto imbalance_value = gnc_import_TransInfo_get_dest_value(trans_info); + auto other_split = xaccSplitGetOtherSplit(selected_match->split); + if (!gnc_numeric_zero_p(imbalance_value) && other_split) + { + if (xaccSplitGetReconcile(other_split) == NREC) + { + xaccSplitSetValue(other_split, imbalance_value); + auto new_amt = gnc_import_TransInfo_get_dest_value(trans_info); + if (gnc_numeric_zero_p(new_amt)) + { + auto other_split_amount = xaccSplitGetAmount(other_split); + auto price = gnc_numeric_div(match_split_amount, other_split_amount, + GNC_DENOM_AUTO, + GNC_HOW_RND_ROUND_HALF_UP); + + new_amt = gnc_numeric_mul(xaccSplitGetAmount(fsplit), price, + GNC_DENOM_AUTO, + GNC_HOW_RND_ROUND_HALF_UP);; + } + xaccSplitSetAmount(other_split, new_amt); + } + else + { + /* else GC will automatically insert a split to equity + to balance the transaction */ + PWARN("Updated transaction '%s', but not other split.", + xaccTransGetDescription(selected_match->trans)); + } + } + + update_desc_and_notes(trans_info); + + /*DEBUG("CommitEdit selected_match")*/ + xaccTransCommitEdit(selected_match->trans); + + process_reconcile (base_acc, trans_info, selected_match); + } + } + return true; + case GNCImport_CLEAR: + { + auto selected_match = gnc_import_TransInfo_get_selected_match (trans_info); + + /* If there is no selection, ignore this transaction. */ + if (!selected_match) + { + PWARN("No matching translaction to be cleared was chosen. Imported transaction will be ignored."); + break; + } + + /* Transaction gets not imported but the matching one gets + reconciled. */ + if (!gnc_import_MatchInfo_get_split (selected_match)) + PERR("The split I am trying to reconcile is nullptr, shouldn't happen!"); + else + { + /* Reconcile the matching transaction */ + process_reconcile(base_acc, trans_info, selected_match); + } + } + return true; + default: + DEBUG("Invalid GNCImportAction for this imported transaction."); + break; + } + /*DEBUG("End");*/ + return false; +} + +/********************************************************************\ + * check_trans_online_id() Callback function used by + * gnc_import_exists_online_id. Takes pointers to transaction and split, + * returns 0 if their online_ids do NOT match, or if the split + * belongs to the transaction +\********************************************************************/ +static gint check_trans_online_id(Transaction *trans1, void *user_data) +{ + gchar *online_id1, *online_id2; + gint retval; + + auto split2 = static_cast(user_data); + auto account = xaccSplitGetAccount(split2); + auto split1 = xaccTransFindSplitByAccount(trans1, account); + if (split1 == split2) + return 0; + + /* hack - we really want to iterate over the _splits_ of the account + instead of the transactions */ + g_assert(split1); + + online_id1 = gnc_import_get_split_online_id (split1); + + if (!online_id1 || !*online_id1) + { + g_free (online_id1); + online_id1 = gnc_import_get_trans_online_id (trans1); + } + + online_id2 = gnc_import_get_split_online_id(split2); + + retval = (!online_id1 || !online_id2 || strcmp (online_id1, online_id2)) ? 0 : 1; + + g_free (online_id1); + g_free (online_id2); + return retval; +} + +static GHashTable* +hash_account_online_ids (Account *account) +{ + auto acct_hash = g_hash_table_new_full + (g_str_hash, g_str_equal, g_free, nullptr); + for (GList *n = xaccAccountGetSplitList (account) ; n; n = n->next) + { + auto id = gnc_import_get_split_online_id (static_cast(n->data)); + if (id && *id) + g_hash_table_insert (acct_hash, (void*) id, GINT_TO_POINTER (1)); + } + return acct_hash; +} + +/** Checks whether the given transaction's online_id already exists in + its parent account. */ +gboolean gnc_import_exists_online_id (Transaction *trans, GHashTable* acct_id_hash) +{ + + /* Look for an online_id in the first split */ + auto source_split = xaccTransGetSplit(trans, 0); + g_assert(source_split); + + auto source_online_id = gnc_import_get_split_online_id (source_split); + + // No online id, no point in continuing. We'd crash if we tried. + if (!source_online_id) + return false; + + // Create a hash per account of a hash of all split IDs. Then the + // test below will be fast if we have many transactions to import. + auto dest_acct = xaccSplitGetAccount (source_split); + if (!g_hash_table_contains (acct_id_hash, dest_acct)) + g_hash_table_insert (acct_id_hash, dest_acct, + hash_account_online_ids (dest_acct)); + auto online_id_exists = g_hash_table_contains ( + static_cast(g_hash_table_lookup (acct_id_hash, dest_acct)), + source_online_id); + + /* If it does, abort the process for this transaction, since it is + already in the system. */ + if (online_id_exists) + { + DEBUG("%s", "Transaction with same online ID exists, destroying current transaction"); + xaccTransDestroy(trans); + xaccTransCommitEdit(trans); + } + g_free (source_online_id); + return online_id_exists; +} + + +/* ****************************************************************** + */ + +/* Calculate lsplit_amount based on knowledge gathered so far + * If insufficient info is available (eg multi currency transaction with missing + * exchange rate provided), set amount to 0 */ +static void trans_info_calculate_dest_amount (GNCImportTransInfo *info) +{ + info->lsplit_value = gnc_numeric_neg (xaccTransGetImbalanceValue (info->trans)); + info->lsplit_amount = {0, 1}; + if (info->dest_acc) + { + auto tcurr = xaccTransGetCurrency(info->trans); + auto dcurr = xaccAccountGetCommodity(info->dest_acc); + + if (gnc_numeric_zero_p(info->lsplit_value)) + return; + + if (gnc_commodity_equiv(tcurr, dcurr)) + info->lsplit_amount = info->lsplit_value; + else if (gnc_numeric_check(info->lsplit_price) == 0) + { + /* We are in a multi currency situation and have a valid price */ + info->lsplit_amount = gnc_numeric_mul (info->lsplit_value, + info->lsplit_price, + GNC_DENOM_AUTO, + GNC_HOW_RND_ROUND_HALF_UP); + } + } +} + +/** Create a new object of GNCImportTransInfo here. */ +GNCImportTransInfo * +gnc_import_TransInfo_new (Transaction *trans, Account *base_acc) +{ + g_assert (trans); + + auto t_info = g_new0(GNCImportTransInfo, 1); + + t_info->trans = trans; + /* Only use first split, the source split */ + auto split = xaccTransGetSplit(trans, 0); + g_assert(split); + t_info->first_split = split; + + /* Try to find a previously selected destination account + string match for the ADD action */ + gnc_import_TransInfo_set_destacc (t_info, + matchmap_find_destination (base_acc, t_info), + false); + + return t_info; +} + + +/** compare_probability() is used by g_list_sort to sort by probability */ +static gint compare_probability (gconstpointer a, + gconstpointer b) +{ + return(((GNCImportMatchInfo *)b)->probability - + ((GNCImportMatchInfo *)a)->probability); +} + +/** Iterates through all splits of trans_info's originating account + * match list. Sorts the resulting list and sets the selected_match + * and action fields in the trans_info. + */ +void +gnc_import_TransInfo_init_matches (GNCImportTransInfo *trans_info, + GNCImportSettings *settings) +{ + g_assert (trans_info); + + if (trans_info->match_list) + { + trans_info->match_list = g_list_sort(trans_info->match_list, + compare_probability); + auto best_match = static_cast(g_list_nth_data(trans_info->match_list, 0)); + gnc_import_TransInfo_set_selected_match_info (trans_info, best_match, false); + if (best_match && + best_match->probability >= gnc_import_Settings_get_clear_threshold(settings)) + { + if (gnc_import_Settings_get_action_update_enabled(settings) && + best_match->update_proposed) + trans_info->action = GNCImport_UPDATE; + else + trans_info->action = GNCImport_CLEAR; + } + else if (!best_match || + best_match->probability <= gnc_import_Settings_get_add_threshold(settings)) + trans_info->action = GNCImport_ADD; + else if (gnc_import_Settings_get_action_skip_enabled(settings)) + trans_info->action = GNCImport_SKIP; + else if (gnc_import_Settings_get_action_update_enabled(settings)) + trans_info->action = GNCImport_UPDATE; + else + trans_info->action = GNCImport_ADD; + } + else + trans_info->action = GNCImport_ADD; + + + trans_info->previous_action = trans_info->action; +} + + +/** @} */ diff --git a/gnucash/import-export/import-backend.h b/gnucash/import-export/import-backend.h index d19d03a165..33f3d5a81b 100644 --- a/gnucash/import-export/import-backend.h +++ b/gnucash/import-export/import-backend.h @@ -46,6 +46,15 @@ typedef struct _matchinfo gboolean update_proposed; } GNCImportMatchInfo; +typedef struct _lsplitinfo +{ + gnc_numeric price; + const char *action; + const char *memo; + char rec_state; + time64 rec_date; +} GNCImportLastSplitInfo; + typedef enum _action { GNCImport_SKIP, @@ -109,20 +118,20 @@ gnc_import_TransInfo_init_matches (GNCImportTransInfo *trans_info, * and processes each ImportTransInfo according to its selected action: * For GNCImport_ADD, the transaction is added etc. etc. * - * Each successful match is also stored in the given ImportMatchMap, - * or, if that argument is NULL, in the ImportMatchMap of each + * Each successful match is also stored in the match map of the given + * account, or if that argument is NULL, in the match map of each * originating account. * - * @param matchmap The ImportMatchMap where each match should be - * stored. May be NULL, in which case the ImportMatchMap of each - * account will be used. + * @param base_acc The account where each match should be + * stored. May be NULL, in which case each originating account + * will be used. * * @param trans_info The ImportTransInfo item to process. * * @return TRUE if the item has been processed. */ gboolean -gnc_import_process_trans_item (GncImportMatchMap *matchmap, +gnc_import_process_trans_item (Account *base_acc, GNCImportTransInfo *trans_info); /** This function generates a new pixmap representing a match score. @@ -154,17 +163,16 @@ GdkPixbuf* gen_probability_pixbuf (gint score, /** Allocates a new TransInfo object, with the Transaction 'trans' * already stored in there. Also, this already checks the - * ImportMatchMap for automated destination account matching. The - * given MatchMap may be NULL, in which case the ImportMatchMap of the + * account's match map for automated destination account matching. The + * given account may be NULL, in which case the match map of the * originating account will be used. * * @param trans The transaction that this TransInfo should work with. * - * @param matchmap MatchMap used for automated destination account - * choosing. This may be NULL, in which case the MatchMap of the + * @param base_acc Account that will provide the match map to lookup a destination + * account. This may be NULL, in which case the match map of the * originating account will be used. */ -GNCImportTransInfo * -gnc_import_TransInfo_new (Transaction *trans, GncImportMatchMap *matchmap); +GNCImportTransInfo* gnc_import_TransInfo_new(Transaction* trans, Account* base_acc); /** Destructor */ void gnc_import_TransInfo_delete (GNCImportTransInfo *info); @@ -223,11 +231,6 @@ gnc_import_TransInfo_set_destacc (GNCImportTransInfo *info, Account *acc, gboolean selected_manually); -/** Try to automatch a given transaction to a destination account */ -gboolean -gnc_import_TransInfo_refresh_destacc (GNCImportTransInfo *transaction_info, - GncImportMatchMap *matchmap); - /** Returns if the currently selected destination account for auto-matching was selected by the user. */ gboolean gnc_import_TransInfo_get_destacc_selected_manually (const GNCImportTransInfo *info); @@ -242,6 +245,28 @@ void gnc_import_TransInfo_set_ref_id (GNCImportTransInfo *info, guint32 ref_id); +/** Returns the exchange rate for this TransInfo. */ +gnc_numeric +gnc_import_TransInfo_get_price (const GNCImportTransInfo *info); + +/** Set the exchange rate for this TransInfo. */ +void +gnc_import_TransInfo_set_price (GNCImportTransInfo *info, + gnc_numeric lprice); + +/** Returns the destination split amount for this TransInfo. */ +gnc_numeric +gnc_import_TransInfo_get_dest_amount (const GNCImportTransInfo *info); + +/** Returns the destination split value for this TransInfo. */ +gnc_numeric +gnc_import_TransInfo_get_dest_value (const GNCImportTransInfo *info); + +/** Sets additional parameters to be used to generate the closing split */ +void +gnc_import_TransInfo_set_last_split_info (GNCImportTransInfo *info, + GNCImportLastSplitInfo *lsplit); + /** Set the append_text for this TransInfo. */ void gnc_import_TransInfo_set_append_text (GNCImportTransInfo *info, diff --git a/gnucash/import-export/import-main-matcher.c b/gnucash/import-export/import-main-matcher.c index 77ae800a9e..6cbf35bbc5 100644 --- a/gnucash/import-export/import-main-matcher.c +++ b/gnucash/import-export/import-main-matcher.c @@ -806,7 +806,6 @@ gnc_gen_trans_assign_transfer_account (GtkTreeView *treeview, GtkTreeIter iter; GNCImportTransInfo *trans_info; Account *old_acc; - gboolean ok_pressed; gchar *path_str = gtk_tree_path_to_string (path); gchar *acct_str = gnc_get_account_name_for_register (*new_acc); @@ -832,12 +831,10 @@ gnc_gen_trans_assign_transfer_account (GtkTreeView *treeview, case GNCImport_ADD: if (gnc_import_TransInfo_is_balanced (trans_info) == FALSE) { - ok_pressed = TRUE; old_acc = gnc_import_TransInfo_get_destacc (trans_info); if (*first) { gchar *acc_full_name; - ok_pressed = FALSE; *new_acc = gnc_import_select_account (info->main_widget, NULL, TRUE, @@ -846,13 +843,13 @@ gnc_gen_trans_assign_transfer_account (GtkTreeView *treeview, gnc_import_TransInfo_get_trans (trans_info)), ACCT_TYPE_NONE, old_acc, - &ok_pressed); + NULL); *first = FALSE; acc_full_name = gnc_account_get_full_name (*new_acc); DEBUG("account selected = %s", acc_full_name); g_free (acc_full_name); } - if (ok_pressed) + if (*new_acc) { gnc_import_TransInfo_set_destacc (trans_info, *new_acc, TRUE); defer_bal_computation (info, *new_acc); @@ -1228,14 +1225,12 @@ gnc_gen_trans_row_activated_cb (GtkTreeView *treeview, GtkTreeViewColumn *column, GNCImportMainMatcher *info) { - Account *assigned_account; - gboolean first, is_selection; + Account *assigned_account = NULL; + gboolean first = TRUE; + gboolean is_selection = FALSE; gchar *namestr; ENTER(""); - assigned_account = NULL; - first = TRUE; - is_selection = FALSE; gnc_gen_trans_assign_transfer_account (treeview, &first, is_selection, path, &assigned_account, info); @@ -1595,9 +1590,6 @@ gnc_gen_trans_init_view (GNCImportMainMatcher *info, G_CALLBACK(gnc_gen_trans_onButtonPressed_cb), info); g_signal_connect (view, "popup-menu", G_CALLBACK(gnc_gen_trans_onPopupMenu_cb), info); - - info->acct_id_hash = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, - (GDestroyNotify)g_hash_table_destroy); } static void @@ -1679,14 +1671,16 @@ gnc_gen_trans_common_setup (GNCImportMainMatcher *info, // Create the checkbox, but do not show it unless there are transactions info->reconcile_after_close = GTK_WIDGET(gtk_builder_get_object (builder, "reconcile_after_close_button")); - show_update = gnc_import_Settings_get_action_update_enabled (info->user_settings); - gnc_gen_trans_init_view (info, all_from_same_account, show_update); - heading_label = GTK_WIDGET(gtk_builder_get_object (builder, "heading_label")); - g_assert (heading_label != NULL); + heading_label = GTK_WIDGET(gtk_builder_get_object (builder, "heading_label")); if (heading) gtk_label_set_text (GTK_LABEL(heading_label), heading); + show_update = gnc_import_Settings_get_action_update_enabled (info->user_settings); + gnc_gen_trans_init_view (info, all_from_same_account, show_update); + + info->acct_id_hash = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, + (GDestroyNotify)g_hash_table_destroy); info->desc_hash = g_hash_table_new (g_str_hash, g_str_equal); info->notes_hash = g_hash_table_new (g_str_hash, g_str_equal); info->memo_hash = g_hash_table_new (g_str_hash, g_str_equal); @@ -1930,7 +1924,7 @@ refresh_model_row (GNCImportMainMatcher *gui, { GtkTreeStore *store; GtkTreeSelection *selection; - gchar *tmp, *imbalance, *text; + gchar *text; const gchar *ro_text, *color = NULL; gchar *int_required_class, *int_prob_required_class, *int_not_required_class; gchar *class_extension = NULL; @@ -1999,55 +1993,69 @@ refresh_model_row (GNCImportMainMatcher *gui, switch (gnc_import_TransInfo_get_action (info)) { case GNCImport_ADD: - if (gnc_import_TransInfo_is_balanced (info) == TRUE) + if (gnc_import_TransInfo_is_balanced (info)) { ro_text = _("New, already balanced"); color = get_required_color (int_not_required_class); } else { - /* Assume that importers won't create transactions in two or more - currencies so we can use xaccTransGetImbalanceValue */ - imbalance = - g_strdup - (xaccPrintAmount - (gnc_numeric_neg (xaccTransGetImbalanceValue - (gnc_import_TransInfo_get_trans (info))), - gnc_commodity_print_info - (xaccTransGetCurrency (gnc_import_TransInfo_get_trans (info)), - TRUE))); - if (gnc_import_TransInfo_get_destacc (info) != NULL) + Account *dest_acc = gnc_import_TransInfo_get_destacc (info); + char *imbalance = NULL; + if (dest_acc) { - color = get_required_color (int_not_required_class); - tmp = gnc_account_get_full_name - (gnc_import_TransInfo_get_destacc (info)); - if (gnc_import_TransInfo_get_destacc_selected_manually (info) - == TRUE) + char *acct_full_name = gnc_account_get_full_name (dest_acc); + gnc_numeric bal_amt = gnc_import_TransInfo_get_dest_amount (info); + if (!gnc_numeric_zero_p (bal_amt)) { - text = - /* Translators: %1$s is the amount to be transferred, - %2$s the destination account. */ - g_strdup_printf (_("New, transfer %s to (manual) \"%s\""), - imbalance, tmp); + GNCPrintAmountInfo pinfo = gnc_commodity_print_info ( + xaccAccountGetCommodity (dest_acc), TRUE); + imbalance = g_strdup (xaccPrintAmount (bal_amt, pinfo)); + color = get_required_color (int_not_required_class); + if (gnc_import_TransInfo_get_destacc_selected_manually (info)) + { + text = + /* Translators: %1$s is the amount to be transferred, + %2$s the destination account. */ + g_strdup_printf (_("New, transfer %s to (manual) \"%s\""), + imbalance, acct_full_name); + } + else + { + text = + /* Translators: %1$s is the amount to be transferred, + %2$s the destination account. */ + g_strdup_printf (_("New, transfer %s to (auto) \"%s\""), + imbalance, acct_full_name); + } } else { + GNCPrintAmountInfo pinfo = gnc_commodity_print_info ( + xaccTransGetCurrency (gnc_import_TransInfo_get_trans (info)), TRUE); + gnc_numeric bal_val = gnc_import_TransInfo_get_dest_value (info); + imbalance = g_strdup (xaccPrintAmount (bal_val, pinfo)); + color = get_required_color (int_required_class); text = - /* Translators: %1$s is the amount to be transferred, - %2$s the destination account. */ - g_strdup_printf (_("New, transfer %s to (auto) \"%s\""), - imbalance, tmp); - } - g_free (tmp); + /* Translators: %s is the amount to be transferred. */ + g_strdup_printf (_("New, UNBALANCED (need price to transfer %s to acct %s)!"), + imbalance, acct_full_name); + } + + g_free (acct_full_name); } else { - color = get_required_color (int_prob_required_class); + GNCPrintAmountInfo pinfo = gnc_commodity_print_info ( + xaccTransGetCurrency (gnc_import_TransInfo_get_trans (info)), TRUE); + gnc_numeric bal_val = gnc_import_TransInfo_get_dest_value (info); + imbalance = g_strdup (xaccPrintAmount (bal_val, pinfo)); + color = get_required_color (int_required_class); text = /* Translators: %s is the amount to be transferred. */ g_strdup_printf (_("New, UNBALANCED (need acct to transfer %s)!"), - imbalance); + imbalance); } remove_child_row (model, iter); @@ -2221,41 +2229,49 @@ gnc_gen_trans_list_get_reconcile_after_close_button (GNCImportMainMatcher *info) } -void -gnc_gen_trans_list_add_trans (GNCImportMainMatcher *gui, Transaction *trans) -{ - Account* acc = NULL; - Split* split = NULL; - int i=0; - - split = xaccTransGetSplit (trans, 0); - acc = xaccSplitGetAccount (split); - defer_bal_computation (gui, acc); - - gnc_gen_trans_list_add_trans_with_ref_id (gui, trans, 0); - return; -}/* end gnc_import_add_trans() */ - -void -gnc_gen_trans_list_add_trans_with_ref_id (GNCImportMainMatcher *gui, Transaction *trans, guint32 ref_id) +static void +gnc_gen_trans_list_add_trans_internal (GNCImportMainMatcher *gui, Transaction *trans, + guint32 ref_id, GNCImportLastSplitInfo* lsplit) { GNCImportTransInfo * transaction_info = NULL; - GtkTreeModel *model; - GtkTreeIter iter; + Account* acc = NULL; + Split* split = NULL; + g_assert (gui); g_assert (trans); if (gnc_import_exists_online_id (trans, gui->acct_id_hash)) return; - else - { - transaction_info = gnc_import_TransInfo_new (trans, NULL); - gnc_import_TransInfo_set_ref_id (transaction_info, ref_id); - // It's much faster to gather the imported transactions into a GSList than directly into the - // treeview. - gui->temp_trans_list = g_slist_prepend (gui->temp_trans_list, transaction_info); - } - return; + + split = xaccTransGetSplit (trans, 0); + acc = xaccSplitGetAccount (split); + defer_bal_computation (gui, acc); + + transaction_info = gnc_import_TransInfo_new (trans, NULL); + gnc_import_TransInfo_set_ref_id (transaction_info, ref_id); + gnc_import_TransInfo_set_last_split_info (transaction_info, lsplit); + // It's much faster to gather the imported transactions into a GSList than + // directly into the treeview. + gui->temp_trans_list = g_slist_prepend (gui->temp_trans_list, transaction_info); +} + +void +gnc_gen_trans_list_add_trans (GNCImportMainMatcher *gui, Transaction *trans) +{ + gnc_gen_trans_list_add_trans_internal (gui, trans, 0, NULL); +} + +void +gnc_gen_trans_list_add_trans_with_ref_id (GNCImportMainMatcher *gui, Transaction *trans, guint32 ref_id) +{ + gnc_gen_trans_list_add_trans_internal (gui, trans, ref_id, NULL); +} + +void gnc_gen_trans_list_add_trans_with_split_data (GNCImportMainMatcher *gui, + Transaction *trans, + GNCImportLastSplitInfo *lsplit) +{ + gnc_gen_trans_list_add_trans_internal (gui, trans, 0, lsplit); } /* Query the accounts used by the imported transactions to find a list of @@ -2320,6 +2336,10 @@ create_hash_of_potential_matches (GList *candidate_txns, GSList* split_list; if (gnc_import_split_has_online_id (candidate->data)) continue; + /* In this context an open transaction represents a freshly + * downloaded one. That can't possibly be a match yet */ + if (xaccTransIsOpen(xaccSplitGetParent(candidate->data))) + continue; split_account = xaccSplitGetAccount (candidate->data); /* g_hash_table_steal_extended would do the two calls in one shot but is * not available until GLib 2.58. diff --git a/gnucash/import-export/import-main-matcher.h b/gnucash/import-export/import-main-matcher.h index 87eff3f39c..651442a2ad 100644 --- a/gnucash/import-export/import-main-matcher.h +++ b/gnucash/import-export/import-main-matcher.h @@ -159,6 +159,25 @@ void gnc_gen_trans_list_delete (GNCImportMainMatcher *info); void gnc_gen_trans_list_add_trans (GNCImportMainMatcher *gui, Transaction *trans); +/** Add a newly imported Transaction to the Transaction Importer. + * The Importer takes over ownership of the passed transaction. + * + * @param gui The Transaction Importer to use. + * + * @param trans The Transaction to add. The must contain at least one + * split, and this split must have been associated with an account + * Only the first split will be used for matching. The transaction + * must NOT be committed. The Importer takes over ownership of the + * passed transaction. + * + * @param lsplit Struct with additional parameters that may be used to + * generate the final split. + */ +void gnc_gen_trans_list_add_trans_with_split_data (GNCImportMainMatcher *gui, + Transaction *trans, + GNCImportLastSplitInfo *lsplit); + + /** Add a newly imported Transaction to the Transaction Importer and provide an * external reference id for it. * The Importer takes over ownership of the passed transaction. diff --git a/gnucash/import-export/test/CMakeLists.txt b/gnucash/import-export/test/CMakeLists.txt index 7ac4fc9696..ccc21d263e 100644 --- a/gnucash/import-export/test/CMakeLists.txt +++ b/gnucash/import-export/test/CMakeLists.txt @@ -59,7 +59,7 @@ set(gtest_import_backend_LIBS set(gtest_import_backend_SOURCES gtest-import-backend.cpp - ${CMAKE_SOURCE_DIR}/gnucash/import-export/import-backend.c + ${CMAKE_SOURCE_DIR}/gnucash/import-export/import-backend.cpp ${CMAKE_SOURCE_DIR}/gnucash/import-export/import-settings.c ${CMAKE_SOURCE_DIR}/gnucash/import-export/import-utilities.c ${CMAKE_SOURCE_DIR}/libgnucash/engine/mocks/gmock-qofinstance.cpp diff --git a/gnucash/import-export/test/gtest-import-backend.cpp b/gnucash/import-export/test/gtest-import-backend.cpp index 9c77f1844a..fc19f709ce 100644 --- a/gnucash/import-export/test/gtest-import-backend.cpp +++ b/gnucash/import-export/test/gtest-import-backend.cpp @@ -86,6 +86,17 @@ gnc_get_num_action (const Transaction *trans, const Split *split) else return NULL; } +// fake function from gnc-commodity.c +// this is a simplified version of the original function +gboolean +gnc_commodity_equiv(const gnc_commodity * a, const gnc_commodity * b) +{ + if (a == b) return TRUE; + if (!a || !b) return FALSE; + + return TRUE; +} + /* required fake functions from app-utils sources, which should not be linked to the test application */ @@ -163,7 +174,6 @@ protected: //! Test for function gnc_import_TransInfo_new() TEST_F(ImportBackendTest, CreateTransInfo) { - GncMockImportMatchMap imap(m_import_acc); gchar* online_id; using namespace testing; @@ -180,11 +190,11 @@ TEST_F(ImportBackendTest, CreateTransInfo) .WillByDefault(Return("This is the description")); // function gnc_import_TransInfo_new() should try to find account using the description from the transaction - EXPECT_CALL(imap, find_account(_, StrEq("This is the description"))) + EXPECT_CALL(*m_import_acc, find_account(_, StrEq("This is the description"))) .WillOnce(Return(m_dest_acc)); // call function to be tested - GNCImportTransInfo *trans_info = gnc_import_TransInfo_new(m_trans, &imap); + GNCImportTransInfo *trans_info = gnc_import_TransInfo_new(m_trans, m_import_acc); // check 'trans_info' EXPECT_EQ(gnc_import_TransInfo_get_fsplit(trans_info), m_split); @@ -230,7 +240,6 @@ TEST_F(ImportBackendBayesTest, CreateTransInfo) { using namespace testing; - GncMockImportMatchMap imap(m_import_acc); time64 date(GncDateTime(GncDate(2020, 3, 18))); struct tm *tm_struct; char local_day_of_week[16]; @@ -264,7 +273,7 @@ TEST_F(ImportBackendBayesTest, CreateTransInfo) .WillByDefault(Return(date)); // check tokens created from transaction - EXPECT_CALL(imap, find_account_bayes(AllOf( + EXPECT_CALL(*m_import_acc, find_account_bayes(AllOf( Each(Not(StrEq(""))), // tokens must not be empty strings Each(Not(HasSubstr(" "))), // tokens must not contain separator Not(HasDuplicates()), // tokens must be unique @@ -275,7 +284,7 @@ TEST_F(ImportBackendBayesTest, CreateTransInfo) .WillOnce(Return(m_dest_acc)); // call function to be tested - GNCImportTransInfo *trans_info = gnc_import_TransInfo_new(m_trans, &imap); + GNCImportTransInfo *trans_info = gnc_import_TransInfo_new(m_trans, m_import_acc); // check 'trans_info' EXPECT_EQ(gnc_import_TransInfo_get_fsplit(trans_info), m_split); diff --git a/libgnucash/engine/Account.cpp b/libgnucash/engine/Account.cpp index a17bec2008..cc4f9f61f9 100644 --- a/libgnucash/engine/Account.cpp +++ b/libgnucash/engine/Account.cpp @@ -5706,90 +5706,71 @@ xaccAccountForEachTransaction(const Account *acc, TransactionCallback proc, #define IMAP_FRAME "import-map" #define IMAP_FRAME_BAYES "import-map-bayes" -/* Obtain an ImportMatchMap object from an Account or a Book */ -GncImportMatchMap * -gnc_account_imap_create_imap (Account *acc) -{ - GncImportMatchMap *imap; - - if (!acc) return NULL; - - imap = g_new0(GncImportMatchMap, 1); - - /* Cache the book for easy lookups; store the account/book for - * marking dirtiness - */ - imap->acc = acc; - imap->book = gnc_account_get_book (acc); - - return imap; -} - /* Look up an Account in the map */ Account* -gnc_account_imap_find_account (GncImportMatchMap *imap, +gnc_account_imap_find_account (Account *acc, const char *category, const char *key) { GValue v = G_VALUE_INIT; GncGUID * guid = NULL; Account *retval; - if (!imap || !key) return NULL; + if (!acc || !key) return NULL; std::vector path {IMAP_FRAME}; if (category) path.push_back (category); path.push_back (key); - qof_instance_get_path_kvp (QOF_INSTANCE (imap->acc), &v, path); + qof_instance_get_path_kvp (QOF_INSTANCE (acc), &v, path); if (G_VALUE_HOLDS_BOXED (&v)) guid = (GncGUID*)g_value_get_boxed (&v); - retval = xaccAccountLookup (guid, imap->book); + retval = xaccAccountLookup (guid, gnc_account_get_book(acc)); g_value_unset (&v); return retval; } /* Store an Account in the map */ void -gnc_account_imap_add_account (GncImportMatchMap *imap, +gnc_account_imap_add_account (Account *acc, const char *category, const char *key, - Account *acc) + Account *added_acc) { GValue v = G_VALUE_INIT; - if (!imap || !key || !acc || (strlen (key) == 0)) return; + if (!acc || !key || !added_acc || (strlen (key) == 0)) return; std::vector path {IMAP_FRAME}; if (category) path.emplace_back (category); path.emplace_back (key); g_value_init (&v, GNC_TYPE_GUID); - g_value_set_boxed (&v, xaccAccountGetGUID (acc)); - xaccAccountBeginEdit (imap->acc); - qof_instance_set_path_kvp (QOF_INSTANCE (imap->acc), &v, path); - qof_instance_set_dirty (QOF_INSTANCE (imap->acc)); - xaccAccountCommitEdit (imap->acc); + g_value_set_boxed (&v, xaccAccountGetGUID (added_acc)); + xaccAccountBeginEdit (acc); + qof_instance_set_path_kvp (QOF_INSTANCE (acc), &v, path); + qof_instance_set_dirty (QOF_INSTANCE (acc)); + xaccAccountCommitEdit (acc); g_value_unset (&v); } /* Remove a reference to an Account in the map */ void -gnc_account_imap_delete_account (GncImportMatchMap *imap, +gnc_account_imap_delete_account (Account *acc, const char *category, const char *key) { - if (!imap || !key) return; + if (!acc || !key) return; std::vector path {IMAP_FRAME}; if (category) path.emplace_back (category); path.emplace_back (key); - xaccAccountBeginEdit (imap->acc); - if (qof_instance_has_path_slot (QOF_INSTANCE (imap->acc), path)) + xaccAccountBeginEdit (acc); + if (qof_instance_has_path_slot (QOF_INSTANCE (acc), path)) { - qof_instance_slot_path_delete (QOF_INSTANCE (imap->acc), path); + qof_instance_slot_path_delete (QOF_INSTANCE (acc), path); if (category) - qof_instance_slot_path_delete_if_empty (QOF_INSTANCE (imap->acc), {IMAP_FRAME, category}); - qof_instance_slot_path_delete_if_empty (QOF_INSTANCE (imap->acc), {IMAP_FRAME}); + qof_instance_slot_path_delete_if_empty (QOF_INSTANCE (acc), {IMAP_FRAME, category}); + qof_instance_slot_path_delete_if_empty (QOF_INSTANCE (acc), {IMAP_FRAME}); } - qof_instance_set_dirty (QOF_INSTANCE (imap->acc)); - xaccAccountCommitEdit (imap->acc); + qof_instance_set_dirty (QOF_INSTANCE (acc)); + xaccAccountCommitEdit (acc); } /*-------------------------------------------------------------------------- @@ -5876,7 +5857,7 @@ highest_probability(FinalProbabilityVec const & probabilities) } static ProbabilityVec -get_first_pass_probabilities(GncImportMatchMap * imap, GList * tokens) +get_first_pass_probabilities(Account* acc, GList * tokens) { ProbabilityVec ret; /* find the probability for each account that contains any of the tokens @@ -5885,7 +5866,7 @@ get_first_pass_probabilities(GncImportMatchMap * imap, GList * tokens) { TokenAccountsInfo tokenInfo{}; auto path = std::string{IMAP_FRAME_BAYES "/"} + static_cast (current_token->data) + "/"; - qof_instance_foreach_slot_prefix (QOF_INSTANCE (imap->acc), path, &build_token_info, tokenInfo); + qof_instance_foreach_slot_prefix (QOF_INSTANCE (acc), path, &build_token_info, tokenInfo); for (auto const & current_account_token : tokenInfo.accounts) { auto item = std::find_if(ret.begin(), ret.end(), [¤t_account_token] @@ -6078,12 +6059,13 @@ static constexpr double threshold = .90 * probability_factor; /* 90% */ /** Look up an Account in the map */ Account* -gnc_account_imap_find_account_bayes (GncImportMatchMap *imap, GList *tokens) +gnc_account_imap_find_account_bayes (Account *acc, GList *tokens) { - if (!imap) + if (!acc) return nullptr; - check_import_map_data (imap->book); - auto first_pass = get_first_pass_probabilities(imap, tokens); + auto book = gnc_account_get_book(acc); + check_import_map_data (book); + auto first_pass = get_first_pass_probabilities(acc, tokens); if (!first_pass.size()) return nullptr; auto final_probabilities = build_probabilities(first_pass); @@ -6100,25 +6082,25 @@ gnc_account_imap_find_account_bayes (GncImportMatchMap *imap, GList *tokens) } catch (gnc::guid_syntax_exception&) { return nullptr; } - auto account = xaccAccountLookup (reinterpret_cast(&guid), imap->book); + auto account = xaccAccountLookup (reinterpret_cast(&guid), book); return account; } static void -change_imap_entry (GncImportMatchMap *imap, std::string const & path, int64_t token_count) +change_imap_entry (Account *acc, std::string const & path, int64_t token_count) { GValue value = G_VALUE_INIT; PINFO("Source Account is '%s', Count is '%" G_GINT64_FORMAT "'", - xaccAccountGetName (imap->acc), token_count); + xaccAccountGetName (acc), token_count); // check for existing guid entry - if (qof_instance_has_slot (QOF_INSTANCE(imap->acc), path.c_str ())) + if (qof_instance_has_slot (QOF_INSTANCE(acc), path.c_str ())) { int64_t existing_token_count = 0; // get the existing_token_count value - qof_instance_get_path_kvp (QOF_INSTANCE (imap->acc), &value, {path}); + qof_instance_get_path_kvp (QOF_INSTANCE (acc), &value, {path}); if (G_VALUE_HOLDS_INT64 (&value)) existing_token_count = g_value_get_int64 (&value); @@ -6134,16 +6116,16 @@ change_imap_entry (GncImportMatchMap *imap, std::string const & path, int64_t to g_value_set_int64 (&value, token_count); // Add or Update the entry based on guid - qof_instance_set_path_kvp (QOF_INSTANCE (imap->acc), &value, {path}); - gnc_features_set_used (imap->book, GNC_FEATURE_GUID_FLAT_BAYESIAN); + qof_instance_set_path_kvp (QOF_INSTANCE (acc), &value, {path}); + gnc_features_set_used (gnc_account_get_book(acc), GNC_FEATURE_GUID_FLAT_BAYESIAN); g_value_unset (&value); } /** Updates the imap for a given account using a list of tokens */ void -gnc_account_imap_add_account_bayes (GncImportMatchMap *imap, +gnc_account_imap_add_account_bayes (Account *acc, GList *tokens, - Account *acc) + Account *added_acc) { GList *current_token; gint64 token_count; @@ -6151,20 +6133,20 @@ gnc_account_imap_add_account_bayes (GncImportMatchMap *imap, char *guid_string; ENTER(" "); - if (!imap) + if (!acc) { LEAVE(" "); return; } - check_import_map_data (imap->book); + check_import_map_data (gnc_account_get_book(acc)); - g_return_if_fail (acc != NULL); - account_fullname = gnc_account_get_full_name(acc); - xaccAccountBeginEdit (imap->acc); + g_return_if_fail (added_acc != NULL); + account_fullname = gnc_account_get_full_name(added_acc); + xaccAccountBeginEdit (acc); PINFO("account name: '%s'", account_fullname); - guid_string = guid_to_string (xaccAccountGetGUID (acc)); + guid_string = guid_to_string (xaccAccountGetGUID (added_acc)); /* process each token in the list */ for (current_token = g_list_first(tokens); current_token; @@ -6181,11 +6163,11 @@ gnc_account_imap_add_account_bayes (GncImportMatchMap *imap, PINFO("adding token '%s'", (char*)current_token->data); auto path = std::string {IMAP_FRAME_BAYES} + '/' + static_cast(current_token->data) + '/' + guid_string; /* change the imap entry for the account */ - change_imap_entry (imap, path, token_count); + change_imap_entry (acc, path, token_count); } /* free up the account fullname and guid string */ - qof_instance_set_dirty (QOF_INSTANCE (imap->acc)); - xaccAccountCommitEdit (imap->acc); + qof_instance_set_dirty (QOF_INSTANCE (acc)); + xaccAccountCommitEdit (acc); g_free (account_fullname); g_free (guid_string); LEAVE(" "); diff --git a/libgnucash/engine/Account.h b/libgnucash/engine/Account.h index cc4fc6bd9f..5a2f2c45d1 100644 --- a/libgnucash/engine/Account.h +++ b/libgnucash/engine/Account.h @@ -68,12 +68,6 @@ typedef struct QofInstanceClass parent_class; } AccountClass; -typedef struct -{ - Account *acc; - QofBook *book; -} GncImportMatchMap; - /* --- type macros --- */ #define GNC_TYPE_ACCOUNT (gnc_account_get_type ()) #define GNC_ACCOUNT(o) \ @@ -914,7 +908,7 @@ typedef enum */ /** The gnc_account_lookup_by_name() subroutine fetches the account by * name from the descendants of the specified account. The immediate - * children are searched first. If there is no match,, then a + * children are searched first. If there is no match, then a * recursive search of all descendants is performed looking for a * match. * @@ -1576,33 +1570,28 @@ typedef enum int xaccAccountTreeForEachTransaction(Account *acc, TransactionCallback proc, void *data); - /** Obtain an ImportMatchMap object from an Account or a Book - */ - GncImportMatchMap *gnc_account_imap_create_imap (Account *acc); - /* Look up an Account in the map non-Baysian */ - Account* gnc_account_imap_find_account (GncImportMatchMap *imap, const char* category, + Account* gnc_account_imap_find_account (Account* acc, const char* category, const char *key); /* Store an Account in the map non Baysian */ - void gnc_account_imap_add_account (GncImportMatchMap *imap, const char *category, - const char *key, Account *acc); + void gnc_account_imap_add_account (Account* acc, const char *category, + const char *key, Account *added_acc); /* Remove a reference to an Account in the map non Baysian */ - void gnc_account_imap_delete_account (GncImportMatchMap *imap, const char *category, - const char *key); + void gnc_account_imap_delete_account(Account* acc, const char* category, const char* key); /** Look up an Account in the map using Baysian */ - Account* gnc_account_imap_find_account_bayes (GncImportMatchMap *imap, GList* tokens); + Account* gnc_account_imap_find_account_bayes (Account* acc, GList* tokens); /** Updates the imap for a given account using a list of tokens */ - void gnc_account_imap_add_account_bayes (GncImportMatchMap *imap, GList* tokens, - Account *acc); + void gnc_account_imap_add_account_bayes (Account* acc, GList* tokens, + Account *added_acc); typedef struct imap_info { diff --git a/libgnucash/engine/mocks/gmock-Account.cpp b/libgnucash/engine/mocks/gmock-Account.cpp index bfed846dae..b85ce2527a 100644 --- a/libgnucash/engine/mocks/gmock-Account.cpp +++ b/libgnucash/engine/mocks/gmock-Account.cpp @@ -51,6 +51,14 @@ gnc_account_get_book(const Account *account) return mockaccount ? mockaccount->get_book() : nullptr; } +gnc_commodity * +xaccAccountGetCommodity(const Account *account) +{ + SCOPED_TRACE(""); + auto mockaccount = gnc_mockaccount(account); + return mockaccount ? mockaccount->get_commodity() : nullptr; +} + gint xaccAccountForEachTransaction(const Account *acc, TransactionCallback proc, void *data) @@ -69,36 +77,30 @@ xaccAccountGetSplitList (const Account *account) } -GncImportMatchMap * -gnc_account_imap_create_imap (Account *acc) -{ - SCOPED_TRACE(""); - auto mockaccount = gnc_mockaccount(acc); - return mockaccount ? mockaccount->create_imap() : nullptr; -} - Account* gnc_account_imap_find_account ( - GncImportMatchMap *imap, + Account *acc, const char* category, const char *key) { - return ((GncMockImportMatchMap*)imap)->find_account(category, key); + auto mockaccount = gnc_mockaccount(acc); + return mockaccount->find_account(category, key); } void gnc_account_imap_add_account ( - GncImportMatchMap *imap, + Account *acc, const char *category, const char *key, - Account *acc) + Account *dest_acc) { - ((GncMockImportMatchMap*)imap)->add_account(category, key, acc); + auto mockaccount = gnc_mockaccount(acc); + mockaccount->add_account(category, key, dest_acc); } Account* gnc_account_imap_find_account_bayes ( - GncImportMatchMap *imap, + Account *acc, GList *tokens) { std::vector tokenVec; @@ -108,14 +110,15 @@ gnc_account_imap_find_account_bayes ( tokenVec.push_back(static_cast (token->data)); } - return ((GncMockImportMatchMap*)imap)->find_account_bayes(tokenVec); + auto mockaccount = gnc_mockaccount(acc); + return mockaccount->find_account_bayes(tokenVec); } void gnc_account_imap_add_account_bayes ( - GncImportMatchMap *imap, + Account *acc, GList *tokens, - Account *acc) + Account *added_acc) { std::vector tokenVec; @@ -124,6 +127,7 @@ gnc_account_imap_add_account_bayes ( tokenVec.push_back(static_cast (token->data)); } - ((GncMockImportMatchMap*)imap)->add_account_bayes(tokenVec, acc); + auto mockaccount = gnc_mockaccount(acc); + mockaccount->add_account_bayes(tokenVec, added_acc); } diff --git a/libgnucash/engine/mocks/gmock-Account.h b/libgnucash/engine/mocks/gmock-Account.h index 07a249b1e2..0fd23ff907 100644 --- a/libgnucash/engine/mocks/gmock-Account.h +++ b/libgnucash/engine/mocks/gmock-Account.h @@ -42,9 +42,13 @@ public: MOCK_METHOD0(begin_edit, void()); MOCK_METHOD0(commit_edit, void()); MOCK_CONST_METHOD0(get_book, QofBook*()); + MOCK_CONST_METHOD0(get_commodity, gnc_commodity*()); MOCK_CONST_METHOD2(for_each_transaction, gint(TransactionCallback, void*)); MOCK_CONST_METHOD0(xaccAccountGetSplitList, SplitList*()); - MOCK_METHOD0(create_imap, GncImportMatchMap*()); + MOCK_METHOD2(find_account, Account *(const char*, const char*)); + MOCK_METHOD3(add_account, void(const char*, const char*, Account*)); + MOCK_METHOD1(find_account_bayes, Account *(std::vector&)); + MOCK_METHOD2(add_account_bayes, void(std::vector&, Account*)); protected: /* Protect destructor to avoid MockAccount objects to be created on stack. MockAccount @@ -53,23 +57,6 @@ protected: }; -// mock up for GncImportMatchMap -class GncMockImportMatchMap : public GncImportMatchMap -{ -public: - GncMockImportMatchMap(MockAccount* account) - { - acc = account; - book = account->get_book(); - }; - - MOCK_METHOD2(find_account, Account *(const char*, const char*)); - MOCK_METHOD3(add_account, void(const char*, const char*, Account*)); - MOCK_METHOD1(find_account_bayes, Account *(std::vector&)); - MOCK_METHOD2(add_account_bayes, void(std::vector&, Account*)); -}; - - // type conversion functions static inline MockAccount* gnc_mockaccount (Account *account) diff --git a/libgnucash/engine/mocks/gmock-Transaction.cpp b/libgnucash/engine/mocks/gmock-Transaction.cpp index 9c92d4ffc2..3229cfe5d2 100644 --- a/libgnucash/engine/mocks/gmock-Transaction.cpp +++ b/libgnucash/engine/mocks/gmock-Transaction.cpp @@ -130,6 +130,14 @@ xaccTransGetNum (const Transaction *trans) return mocktrans ? mocktrans->get_num() : ""; } +gnc_commodity * +xaccTransGetCurrency (const Transaction *trans) +{ + SCOPED_TRACE(""); + auto mocktrans = gnc_mocktransaction(trans); + return mocktrans ? mocktrans->get_currency() : nullptr; +} + gboolean xaccTransIsOpen (const Transaction *trans) { diff --git a/libgnucash/engine/mocks/gmock-Transaction.h b/libgnucash/engine/mocks/gmock-Transaction.h index ce82437fa7..08d5e968ab 100644 --- a/libgnucash/engine/mocks/gmock-Transaction.h +++ b/libgnucash/engine/mocks/gmock-Transaction.h @@ -59,6 +59,7 @@ public: MOCK_METHOD1(set_date_posted_secs_normalized, void(time64)); MOCK_CONST_METHOD0(get_description, const char *()); MOCK_METHOD1(set_description, void(const char*)); + MOCK_CONST_METHOD0(get_currency, gnc_commodity *()); MOCK_CONST_METHOD0(get_notes, const char *()); MOCK_METHOD1(set_notes, void(const char*)); MOCK_CONST_METHOD0(get_imbalance_value, gnc_numeric()); diff --git a/libgnucash/engine/test/gtest-import-map.cpp b/libgnucash/engine/test/gtest-import-map.cpp index a7f83e03ed..598fa7c6fe 100644 --- a/libgnucash/engine/test/gtest-import-map.cpp +++ b/libgnucash/engine/test/gtest-import-map.cpp @@ -81,15 +81,6 @@ protected: Account *t_expense_account {}; }; -TEST_F(ImapTest, CreateImap) { - GncImportMatchMap *imap = gnc_account_imap_create_imap (t_bank_account); - EXPECT_NE(nullptr, imap); - EXPECT_EQ(t_bank_account, imap->acc); - EXPECT_EQ(gnc_account_get_book(t_bank_account), imap->book); - - g_free(imap); -} - static const char* IMAP_FRAME = "import-map"; static const char* IMAP_FRAME_BAYES = "import-map-bayes"; @@ -98,15 +89,14 @@ class ImapPlainTest : public ImapTest protected: void SetUp() { ImapTest::SetUp(); - t_imap = gnc_account_imap_create_imap (t_bank_account); + t_acc = t_bank_account; } void TearDown() { - g_free(t_imap); ImapTest::TearDown(); } - GncImportMatchMap *t_imap {}; + Account *t_acc {}; }; TEST_F(ImapPlainTest, FindAccount) @@ -119,11 +109,11 @@ TEST_F(ImapPlainTest, FindAccount) root->set_path({IMAP_FRAME, "pepper"}, new KvpValue{*acc1_val}); root->set_path({IMAP_FRAME, "salt"}, new KvpValue{*acc2_val}); - EXPECT_EQ(t_expense_account1, gnc_account_imap_find_account(t_imap, "foo", "bar")); - EXPECT_EQ(t_expense_account2, gnc_account_imap_find_account(t_imap, "baz", "waldo")); - EXPECT_EQ(t_expense_account1, gnc_account_imap_find_account(t_imap, NULL, "pepper")); - EXPECT_EQ(t_expense_account2, gnc_account_imap_find_account(t_imap, NULL, "salt")); - EXPECT_EQ(nullptr, gnc_account_imap_find_account(t_imap, "salt", NULL)); + EXPECT_EQ(t_expense_account1, gnc_account_imap_find_account(t_acc, "foo", "bar")); + EXPECT_EQ(t_expense_account2, gnc_account_imap_find_account(t_acc, "baz", "waldo")); + EXPECT_EQ(t_expense_account1, gnc_account_imap_find_account(t_acc, NULL, "pepper")); + EXPECT_EQ(t_expense_account2, gnc_account_imap_find_account(t_acc, NULL, "salt")); + EXPECT_EQ(nullptr, gnc_account_imap_find_account(t_acc, "salt", NULL)); } TEST_F(ImapPlainTest, AddAccount) @@ -131,23 +121,23 @@ TEST_F(ImapPlainTest, AddAccount) // prevent the embedded beginedit/committedit from doing anything qof_instance_increase_editlevel(QOF_INSTANCE(t_bank_account)); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_add_account(t_imap, "foo", "bar", t_expense_account1); - gnc_account_imap_add_account(t_imap, "baz", "waldo", t_expense_account2); - gnc_account_imap_add_account(t_imap, NULL, "pepper", t_expense_account1); - gnc_account_imap_add_account(t_imap, NULL, "salt", t_expense_account2); + gnc_account_imap_add_account(t_acc, "foo", "bar", t_expense_account1); + gnc_account_imap_add_account(t_acc, "baz", "waldo", t_expense_account2); + gnc_account_imap_add_account(t_acc, NULL, "pepper", t_expense_account1); + gnc_account_imap_add_account(t_acc, NULL, "salt", t_expense_account2); EXPECT_EQ(1, qof_instance_get_editlevel(QOF_INSTANCE(t_bank_account))); EXPECT_TRUE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_add_account(t_imap, NULL, NULL, t_expense_account2); + gnc_account_imap_add_account(t_acc, NULL, NULL, t_expense_account2); EXPECT_FALSE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); - gnc_account_imap_add_account(t_imap, "pork", "sausage", NULL); + gnc_account_imap_add_account(t_acc, "pork", "sausage", NULL); EXPECT_FALSE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_reset_editlevel(QOF_INSTANCE(t_bank_account)); auto root = qof_instance_get_slots(QOF_INSTANCE(t_bank_account)); auto value = root->get_slot({IMAP_FRAME, "foo", "bar"}); auto check_account = [this](KvpValue* v) { - return xaccAccountLookup(v->get(), this->t_imap->book); }; + return xaccAccountLookup(v->get(), gnc_account_get_book(this->t_acc)); }; EXPECT_EQ(t_expense_account1, check_account(value)); value = root->get_slot({IMAP_FRAME, "baz", "waldo"}); EXPECT_EQ(t_expense_account2, check_account(value)); @@ -168,30 +158,30 @@ TEST_F(ImapPlainTest, DeleteAccount) // prevent the embedded beginedit/committedit from doing anything qof_instance_increase_editlevel(QOF_INSTANCE(t_bank_account)); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_add_account(t_imap, "foo", "bar", t_expense_account1); - gnc_account_imap_add_account(t_imap, "foo", "waldo", t_expense_account2); - gnc_account_imap_add_account(t_imap, NULL, "pepper", t_expense_account1); + gnc_account_imap_add_account(t_acc, "foo", "bar", t_expense_account1); + gnc_account_imap_add_account(t_acc, "foo", "waldo", t_expense_account2); + gnc_account_imap_add_account(t_acc, NULL, "pepper", t_expense_account1); EXPECT_EQ(1, qof_instance_get_editlevel(QOF_INSTANCE(t_bank_account))); EXPECT_TRUE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_delete_account(t_imap, NULL, NULL); + gnc_account_imap_delete_account(t_acc, NULL, NULL); EXPECT_FALSE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); - gnc_account_imap_delete_account(t_imap, "foo", "waldo"); + gnc_account_imap_delete_account(t_acc, "foo", "waldo"); EXPECT_TRUE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - EXPECT_EQ(t_expense_account1, gnc_account_imap_find_account(t_imap, "foo", "bar")); - EXPECT_EQ(nullptr, gnc_account_imap_find_account(t_imap, "foo", "waldo")); + EXPECT_EQ(t_expense_account1, gnc_account_imap_find_account(t_acc, "foo", "bar")); + EXPECT_EQ(nullptr, gnc_account_imap_find_account(t_acc, "foo", "waldo")); auto root = qof_instance_get_slots(QOF_INSTANCE(t_bank_account)); EXPECT_EQ(nullptr, root->get_slot(path1)); - gnc_account_imap_delete_account(t_imap, "foo", "bar"); + gnc_account_imap_delete_account(t_acc, "foo", "bar"); EXPECT_TRUE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); EXPECT_EQ(nullptr, root->get_slot(path2)); - gnc_account_imap_delete_account(t_imap, NULL, "pepper"); + gnc_account_imap_delete_account(t_acc, NULL, "pepper"); EXPECT_TRUE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); EXPECT_EQ(nullptr, root->get_slot(path3)); @@ -258,23 +248,23 @@ TEST_F(ImapBayesTest, FindAccountBayes) root->set_path({std::string{IMAP_FRAME_BAYES} + "/" + pepper + "/" + acct1_guid}, new KvpValue{*value}); root->set_path({std::string{IMAP_FRAME_BAYES} + "/" + salt + "/" + acct2_guid}, new KvpValue{*value}); - auto account = gnc_account_imap_find_account_bayes(t_imap, t_list1); + auto account = gnc_account_imap_find_account_bayes(t_acc, t_list1); EXPECT_EQ(t_expense_account1, account); - account = gnc_account_imap_find_account_bayes(t_imap, t_list2); + account = gnc_account_imap_find_account_bayes(t_acc, t_list2); EXPECT_EQ(t_expense_account2, account); - account = gnc_account_imap_find_account_bayes(t_imap, t_list3); + account = gnc_account_imap_find_account_bayes(t_acc, t_list3); EXPECT_EQ(t_expense_account1, account); - account = gnc_account_imap_find_account_bayes(t_imap, t_list4); + account = gnc_account_imap_find_account_bayes(t_acc, t_list4); EXPECT_EQ(t_expense_account2, account); - account = gnc_account_imap_find_account_bayes(t_imap, t_list5); + account = gnc_account_imap_find_account_bayes(t_acc, t_list5); EXPECT_EQ(nullptr, account); // only imap entries with exact token matching should be considered root->set_path({std::string{IMAP_FRAME_BAYES} + "/" + pepper + waldo + "/" + acct2_guid}, new KvpValue{*value}); - account = gnc_account_imap_find_account_bayes(t_imap, t_list3); + account = gnc_account_imap_find_account_bayes(t_acc, t_list3); EXPECT_EQ(t_expense_account1, account); root->set_path({std::string{IMAP_FRAME_BAYES} + "/" + pepper + "/" + waldo + "/" + acct2_guid}, new KvpValue{*value}); - account = gnc_account_imap_find_account_bayes(t_imap, t_list3); + account = gnc_account_imap_find_account_bayes(t_acc, t_list3); EXPECT_EQ(t_expense_account1, account); } @@ -283,14 +273,14 @@ TEST_F(ImapBayesTest, AddAccountBayes) // prevent the embedded beginedit/committedit from doing anything qof_instance_increase_editlevel(QOF_INSTANCE(t_bank_account)); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_add_account_bayes(t_imap, t_list1, t_expense_account1); - gnc_account_imap_add_account_bayes(t_imap, t_list2, t_expense_account2); - gnc_account_imap_add_account_bayes(t_imap, t_list3, t_expense_account1); - gnc_account_imap_add_account_bayes(t_imap, t_list4, t_expense_account2); + gnc_account_imap_add_account_bayes(t_acc, t_list1, t_expense_account1); + gnc_account_imap_add_account_bayes(t_acc, t_list2, t_expense_account2); + gnc_account_imap_add_account_bayes(t_acc, t_list3, t_expense_account1); + gnc_account_imap_add_account_bayes(t_acc, t_list4, t_expense_account2); EXPECT_EQ(1, qof_instance_get_editlevel(QOF_INSTANCE(t_bank_account))); EXPECT_TRUE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_add_account_bayes(t_imap, t_list5, NULL); + gnc_account_imap_add_account_bayes(t_acc, t_list5, NULL); EXPECT_FALSE(qof_instance_get_dirty_flag(QOF_INSTANCE(t_bank_account))); qof_instance_reset_editlevel(QOF_INSTANCE(t_bank_account)); @@ -299,7 +289,7 @@ TEST_F(ImapBayesTest, AddAccountBayes) auto acct2_guid = guid_to_string (xaccAccountGetGUID(t_expense_account2)); auto value = root->get_slot({std::string{IMAP_FRAME_BAYES} + "/foo/bar"}); auto check_account = [this](KvpValue* v) { - return (v->get(), this->t_imap->book); }; + return (v->get(), gnc_account_get_book(this->t_acc)); }; value = root->get_slot({std::string{IMAP_FRAME_BAYES} + "/" + foo + "/" + acct1_guid}); EXPECT_EQ(1, value->get()); value = root->get_slot({std::string{IMAP_FRAME_BAYES} + "/" + bar + "/" + acct1_guid}); @@ -316,7 +306,7 @@ TEST_F(ImapBayesTest, AddAccountBayes) EXPECT_EQ(nullptr, value); qof_instance_increase_editlevel(QOF_INSTANCE(t_bank_account)); - gnc_account_imap_add_account_bayes(t_imap, t_list2, t_expense_account2); + gnc_account_imap_add_account_bayes(t_acc, t_list2, t_expense_account2); qof_instance_mark_clean(QOF_INSTANCE(t_bank_account)); qof_instance_reset_editlevel(QOF_INSTANCE(t_bank_account)); value = root->get_slot({std::string{IMAP_FRAME_BAYES} + "/" + baz + "/" + acct2_guid}); @@ -326,7 +316,7 @@ TEST_F(ImapBayesTest, AddAccountBayes) TEST_F(ImapBayesTest, ConvertBayesData) { auto root = qof_instance_get_slots(QOF_INSTANCE(t_bank_account)); - auto book = qof_instance_get_slots(QOF_INSTANCE(t_imap->book)); + auto book = qof_instance_get_slots(QOF_INSTANCE(gnc_account_get_book(this->t_acc))); auto acct1_guid = guid_to_string (xaccAccountGetGUID(t_expense_account1)); //Food auto acct2_guid = guid_to_string (xaccAccountGetGUID(t_expense_account2)); //Drink auto acct3_guid = guid_to_string (xaccAccountGetGUID(t_asset_account2)); //Asset-Bank @@ -345,7 +335,7 @@ TEST_F(ImapBayesTest, ConvertBayesData) /* make sure to reset the conversion has been run flag */ gnc_account_reset_convert_bayes_to_flat (); /*Calling into the imap functions should trigger a conversion.*/ - gnc_account_imap_add_account_bayes(t_imap, t_list5, t_expense_account2); //pork and sausage; account Food + gnc_account_imap_add_account_bayes(t_acc, t_list5, t_expense_account2); //pork and sausage; account Food // convert from 'Asset-Bank' to 'Asset-Bank' guid auto value = root->get_slot({std::string{IMAP_FRAME_BAYES} + "/severely/divided/token/" + acct3_guid}); EXPECT_EQ(10, value->get()); @@ -373,10 +363,10 @@ TEST_F (ImapBayesTest, import_map_with_delimiters) { GList * tokens {nullptr}; tokens = g_list_prepend (tokens, const_cast ("one/two/three")); - gnc_account_imap_add_account_bayes (t_imap, tokens, t_expense_account1); - gnc_account_imap_add_account_bayes (t_imap, tokens, t_expense_account1); - gnc_account_imap_add_account_bayes (t_imap, tokens, t_expense_account1); - auto account = gnc_account_imap_find_account_bayes (t_imap, tokens); + gnc_account_imap_add_account_bayes (t_acc, tokens, t_expense_account1); + gnc_account_imap_add_account_bayes (t_acc, tokens, t_expense_account1); + gnc_account_imap_add_account_bayes (t_acc, tokens, t_expense_account1); + auto account = gnc_account_imap_find_account_bayes (t_acc, tokens); EXPECT_EQ (account, t_expense_account1); } @@ -384,8 +374,8 @@ TEST_F (ImapBayesTest, get_bayes_info) { GList * tokens {nullptr}; tokens = g_list_prepend (tokens, const_cast ("one/two/three")); - gnc_account_imap_add_account_bayes(t_imap, tokens, t_expense_account1); - auto account = gnc_account_imap_find_account_bayes (t_imap, tokens); + gnc_account_imap_add_account_bayes(t_acc, tokens, t_expense_account1); + auto account = gnc_account_imap_find_account_bayes (t_acc, tokens); EXPECT_EQ (account, t_expense_account1); auto infos = gnc_account_imap_get_info_bayes (t_bank_account); EXPECT_EQ (g_list_first (infos), g_list_last (infos)); diff --git a/po/POTFILES.in b/po/POTFILES.in index d7bd709b40..15d2fe8ae8 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -349,7 +349,7 @@ gnucash/import-export/customer-import/dialog-customer-import.c gnucash/import-export/customer-import/dialog-customer-import-gui.c gnucash/import-export/customer-import/gnc-plugin-customer-import.c gnucash/import-export/import-account-matcher.c -gnucash/import-export/import-backend.c +gnucash/import-export/import-backend.cpp gnucash/import-export/import-commodity-matcher.c gnucash/import-export/import-format-dialog.c gnucash/import-export/import-main-matcher.c