From 90e5e96f8fbc50bf702b90cd524176c53ff5d8c7 Mon Sep 17 00:00:00 2001 From: Geert Janssens Date: Tue, 29 Nov 2016 12:32:02 +0100 Subject: [PATCH] Rework the intermediate properties storage The overly complex templated class hierarchy is replaced with two simple classes. One to keep the discovered transaction properties and one for the discovered split properties. Make both classes responsible for verifying it's state and creating the necessary objects. --- src/import-export/csv-imp/CMakeLists.txt | 2 + src/import-export/csv-imp/Makefile.am | 2 + src/import-export/csv-imp/gnc-trans-props.cpp | 419 +++++++++++ src/import-export/csv-imp/gnc-trans-props.hpp | 115 +++ src/import-export/csv-imp/gnc-tx-import.cpp | 653 +++--------------- src/import-export/csv-imp/gnc-tx-import.hpp | 53 +- 6 files changed, 645 insertions(+), 599 deletions(-) create mode 100644 src/import-export/csv-imp/gnc-trans-props.cpp create mode 100644 src/import-export/csv-imp/gnc-trans-props.hpp diff --git a/src/import-export/csv-imp/CMakeLists.txt b/src/import-export/csv-imp/CMakeLists.txt index 72143a2b40..4b5fe585b3 100644 --- a/src/import-export/csv-imp/CMakeLists.txt +++ b/src/import-export/csv-imp/CMakeLists.txt @@ -17,6 +17,7 @@ SET(csv_import_SOURCES gnc-dummy-tokenizer.cpp gnc-fw-tokenizer.cpp gnc-tokenizer.cpp + gnc-trans-props.cpp gnc-tx-import.cpp ${CMAKE_SOURCE_DIR}/lib/stf/stf-parse.c ${CMAKE_SOURCE_DIR}/lib/goffice/go-charmap-sel.c @@ -42,6 +43,7 @@ SET(csv_import_noinst_HEADERS gnc-dummy-tokenizer.hpp gnc-fw-tokenizer.hpp gnc-tokenizer.hpp + gnc-trans-props.hpp gnc-tx-import.hpp ${CMAKE_SOURCE_DIR}/lib/stf/stf-parse.h ${CMAKE_SOURCE_DIR}/lib/goffice/go-charmap-sel.h diff --git a/src/import-export/csv-imp/Makefile.am b/src/import-export/csv-imp/Makefile.am index e3c4775b58..0014c8825c 100644 --- a/src/import-export/csv-imp/Makefile.am +++ b/src/import-export/csv-imp/Makefile.am @@ -18,6 +18,7 @@ libgncmod_csv_import_la_SOURCES = \ gnc-fw-tokenizer.cpp \ gnc-tokenizer.cpp \ gnc-tx-import.cpp \ + gnc-trans-props.cpp \ gnc-csv-trans-settings.c noinst_HEADERS = \ @@ -35,6 +36,7 @@ noinst_HEADERS = \ gnc-fw-tokenizer.hpp \ gnc-tokenizer.hpp \ gnc-tx-import.hpp \ + gnc-trans-props.hpp \ gnc-csv-trans-settings.h libgncmod_csv_import_la_LDFLAGS = -avoid-version diff --git a/src/import-export/csv-imp/gnc-trans-props.cpp b/src/import-export/csv-imp/gnc-trans-props.cpp new file mode 100644 index 0000000000..e85f699e79 --- /dev/null +++ b/src/import-export/csv-imp/gnc-trans-props.cpp @@ -0,0 +1,419 @@ +/********************************************************************\ + * gnc-csv-imp-trans.cpp - import transactions from csv files * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License* + * along with this program; if not, contact: * + * * + * Free Software Foundation Voice: +1-617-542-5942 * + * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 * + * Boston, MA 02110-1301, USA gnu@gnu.org * + * * +\********************************************************************/ + +extern "C" { +#include +#if PLATFORM(WINDOWS) +#include +#endif + +#include + +#include "engine-helpers.h" +#include "gnc-csv-account-map.h" +#include "gnc-ui-util.h" +#include "Account.h" +#include "Transaction.h" + +} + +#include +#include +#include +#include "gnc-trans-props.hpp" + +G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT; + +/* This map contains a set of strings representing the different column types. */ +std::map gnc_csv_col_type_strs = { + { GncTransPropType::NONE, N_("None") }, + { GncTransPropType::DATE, N_("Date") }, + { GncTransPropType::NUM, N_("Num") }, + { GncTransPropType::DESCRIPTION, N_("Description") }, + { GncTransPropType::NOTES, N_("Notes") }, + { GncTransPropType::ACCOUNT, N_("Account") }, + { GncTransPropType::DEPOSIT, N_("Deposit") }, + { GncTransPropType::WITHDRAWAL, N_("Withdrawal") }, + { GncTransPropType::BALANCE, N_("Balance") }, + { GncTransPropType::MEMO, N_("Memo") }, + { GncTransPropType::OACCOUNT, N_("Other Account") }, + { GncTransPropType::OMEMO, N_("Other Memo") } +}; + +/* Regular expressions used to parse dates per date format */ +const char* date_regex[] = { + "(?:" // either y-m-d + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)" + "|" // or CCYYMMDD + "(?[0-9]{4})" + "(?[0-9]{2})" + "(?[0-9]{2})" + ")", + + "(?:" // either d-m-y + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)" + "|" // or DDMMCCYY + "(?[0-9]{2})" + "(?[0-9]{2})" + "(?[0-9]{4})" + ")", + + "(?:" // either m-d-y + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)" + "|" // or MMDDCCYY + "(?[0-9]{2})" + "(?[0-9]{2})" + "(?[0-9]{4})" + ")", + + "(?:" // either d-m(-y) + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)(?:[-/.' ]+" + "(?[0-9]+))?" + "|" // or DDMM(CCYY) + "(?[0-9]{2})" + "(?[0-9]{2})" + "(?[0-9]+)?" + ")", + + "(?:" // either m-d(-y) + "(?[0-9]+)[-/.' ]+" + "(?[0-9]+)(?:[-/.' ]+" + "(?[0-9]+))?" + "|" // or MMDD(CCYY) + "(?[0-9]{2})" + "(?[0-9]{2})" + "(?[0-9]+)?" + ")", +}; + +/** Parses a string into a date, given a format. This function + * requires only knowing the order in which the year, month and day + * appear. For example, 01-02-2003 will be parsed the same way as + * 01/02/2003. + * @param date_str The string containing a date being parsed + * @param format An index specifying a format in date_format_user + * @exception std::invalid_argument if the string can't be parsed into a date. + * @return The parsed value of date_str on success, throws on failure + */ + +time64 parse_date (const std::string &date_str, int format) +{ + boost::regex r(date_regex[format]); + boost::smatch what; + if(!boost::regex_search(date_str, what, r)) + throw std::invalid_argument ("String doesn't appear to be formatted as a date."); // regex didn't find a match + + // Attention: different behavior from 2.6.x series ! + // If date format without year was selected, the match + // should NOT have found a year. + if ((format >= 3) && (what.length("YEAR") != 0)) + throw std::invalid_argument ("String appears to contain a year while the selected format forbids this."); + + auto day = std::stoi (what.str("DAY")); + auto month = std::stoi (what.str("MONTH")); + + int year; + if (format < 3) + { + /* The input dates have a year, so use that one */ + year = std::stoi (what.str("YEAR")); + + /* Handle two-digit years. */ + if (year < 100) + { + /* We allow two-digit years in the range 1969 - 2068. */ + if (year < 69) + year += 2000; + else + year += 1900; + } + } + else + { + /* The input dates don't have a year, so work with today's year. + */ + gnc_timespec2dmy(timespec_now(), nullptr, nullptr, &year); + } + + auto ts = gnc_dmy2timespec_neutral(day, month, year); + return ts.tv_sec; +} + + +/** Convert str into a gnc_numeric using the user-specified (import) currency format. + * @param str The string to be parsed + * @param currency_format The currency format to use. + * @return a gnc_numeric on success, boost::none on failure + */ +static boost::optional convert_amount_col_str (const std::string &str, int currency_format) +{ + /* If a cell is empty or just spaces return invalid amount */ + if(!boost::regex_search(str, boost::regex("[0-9]"))) + throw std::invalid_argument ("String doesn't appear to contain a valid number."); + + auto expr = boost::make_u32regex("[[:Sc:]]"); + std::string str_no_symbols = boost::u32regex_replace(str, expr, ""); + + /* Convert based on user chosen currency format */ + gnc_numeric val; + char *endptr; + switch (currency_format) + { + case 0: + /* Currency locale */ + if (!(xaccParseAmount (str_no_symbols.c_str(), TRUE, &val, &endptr))) + throw std::invalid_argument ("String can't be parsed into a number using the selected currency format."); + break; + case 1: + /* Currency decimal period */ + if (!(xaccParseAmountExtended (str_no_symbols.c_str(), TRUE, '-', '.', ',', "\003\003", "$+", &val, &endptr))) + throw std::invalid_argument ("String can't be parsed into a number using the selected currency format."); + break; + case 2: + /* Currency decimal comma */ + if (!(xaccParseAmountExtended (str_no_symbols.c_str(), TRUE, '-', ',', '.', "\003\003", "$+", &val, &endptr))) + throw std::invalid_argument ("String can't be parsed into a number using the selected currency format."); + break; + } + + return val; +} + + +void GncPreTrans::set_property (GncTransPropType prop_type, const std::string& prop_value_str, int date_format) +{ + switch (prop_type) + { + case GncTransPropType::DATE: + m_date = parse_date (prop_value_str.c_str(), date_format); // Throws if parsing fails + break; + + case GncTransPropType::DESCRIPTION: + if (!prop_value_str.empty()) + m_desc = prop_value_str; + else + m_desc = boost::none; + break; + + case GncTransPropType::NOTES: + if (!prop_value_str.empty()) + m_notes = prop_value_str; + else + m_notes = boost::none; + break; + + default: + /* Issue a warning for all other prop_types. */ + PWARN ("%d is an invalid property for a transaction", static_cast(prop_type)); + break; + } + +} + +std::string GncPreTrans::verify_essentials (void) +{ + /* Make sure this transaction has the minimum required set of properties defined */ + if (!m_date) + return N_("No date column."); + else + return std::string(); +} + +Transaction* GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency) +{ + if (created) + return nullptr; + + auto trans = xaccMallocTransaction (book); + xaccTransBeginEdit (trans); + xaccTransSetCurrency (trans, currency); + + xaccTransSetDatePostedSecsNormalized (trans, *m_date); + + if (m_desc) + xaccTransSetDescription (trans, m_desc->c_str()); + + if (m_notes) + xaccTransSetNotes (trans, m_notes->c_str()); + + created = true; + return trans; +} + + +void GncPreSplit::set_property (GncTransPropType prop_type, const std::string& prop_value_str, int currency_format) +{ + Account *acct = nullptr; + switch (prop_type) + { + case GncTransPropType::ACCOUNT: + acct = gnc_csv_account_map_search (prop_value_str.c_str()); + if (acct) + m_account = acct; + else + throw std::invalid_argument ("String can't be mapped back to an account."); + break; + + case GncTransPropType::OACCOUNT: + acct = gnc_csv_account_map_search (prop_value_str.c_str()); + if (acct) + m_oaccount = acct; + else + throw std::invalid_argument ("String can't be mapped back to an account."); + break; + + case GncTransPropType::MEMO: + if (!prop_value_str.empty()) + m_memo = prop_value_str; + else + m_memo = boost::none; + break; + + case GncTransPropType::OMEMO: + if (!prop_value_str.empty()) + m_omemo = prop_value_str; + else + m_omemo = boost::none; + break; + + case GncTransPropType::NUM: + if (!prop_value_str.empty()) + m_num = prop_value_str; + else + m_num = boost::none; + break; + + case GncTransPropType::BALANCE: + m_balance = convert_amount_col_str (prop_value_str, currency_format); // Will throw if parsing fails + break; + case GncTransPropType::DEPOSIT: + m_deposit = convert_amount_col_str (prop_value_str, currency_format); // Will throw if parsing fails + break; + case GncTransPropType::WITHDRAWAL: + m_withdrawal = convert_amount_col_str (prop_value_str, currency_format); // Will throw if parsing fails + break; + + default: + /* Issue a warning for all other prop_types. */ + PWARN ("%d is an invalid property for a split", static_cast(prop_type)); + break; + } + +} + +std::string GncPreSplit::verify_essentials (void) +{ + /* Make sure this split has the minimum required set of properties defined. */ + if (!m_deposit && !m_withdrawal && !m_balance) + return N_("No balance, deposit, or withdrawal column."); + else + return std::string(); +} + +/** Adds a split to a transaction. + * @param trans The transaction to add a split to + * @param account The account used for the split + * @param book The book where the split should be stored + * @param amount The amount of the split + */ +static void trans_add_split (Transaction* trans, Account* account, QofBook* book, + gnc_numeric amount, const std::string& num, const std::string& memo) +{ + auto split = xaccMallocSplit (book); + xaccSplitSetAccount (split, account); + xaccSplitSetParent (split, trans); + xaccSplitSetAmount (split, amount); + xaccSplitSetValue (split, amount); + if (!memo.empty()) + xaccSplitSetMemo (split, memo.c_str()); + /* set tran-num and/or split-action per book option + * note this function does nothing if num is NULL also */ + if (!num.empty()) + gnc_set_num_action (trans, split, num.c_str(), NULL); +} + +boost::optional GncPreSplit::create_split (Transaction* trans) +{ + if (created) + return boost::none; + + auto book = xaccTransGetBook (trans); + std::string num; + std::string memo; + std::string omemo; + Account *account = nullptr; + Account *oaccount = nullptr; + bool amount_set = false; + gnc_numeric deposit = { 0, 1 }; + gnc_numeric withdrawal = { 0, 1 }; + gnc_numeric amount = { 0, 1 }; + + if (m_account) + account = *m_account; + if (m_oaccount) + oaccount = *m_oaccount; + if (m_memo) + memo = *m_memo; + if (m_omemo) + omemo = *m_omemo; + if (m_num) + num = *m_num; + if (m_deposit) + { + deposit = *m_deposit; + amount_set = true; + } + if (m_withdrawal) + { + withdrawal = *m_withdrawal; + amount_set = true; + } + if (amount_set) + amount = gnc_numeric_add (deposit, withdrawal, + xaccAccountGetCommoditySCU (account), + GNC_HOW_RND_ROUND_HALF_UP); + + /* Add a split with the cumulative amount value. */ + trans_add_split (trans, account, book, amount, num, memo); + + if (oaccount) + /* Note: the current importer assumes at most 2 splits. This means the second split amount + * will be the negative of the the first split amount. We also only set the num field once, + * for the first split. + */ + trans_add_split (trans, oaccount, book, gnc_numeric_neg(amount), "", omemo); + + + created = true; + + if (amount_set) + return boost::none; + else + return m_balance; +} diff --git a/src/import-export/csv-imp/gnc-trans-props.hpp b/src/import-export/csv-imp/gnc-trans-props.hpp new file mode 100644 index 0000000000..7b1552df5c --- /dev/null +++ b/src/import-export/csv-imp/gnc-trans-props.hpp @@ -0,0 +1,115 @@ +/********************************************************************\ + * gnc-trans-props.hpp - encapsulate transaction properties for use * + * in the csv importer * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License* + * along with this program; if not, contact: * + * * + * Free Software Foundation Voice: +1-617-542-5942 * + * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 * + * Boston, MA 02110-1301, USA gnu@gnu.org * + * * +\********************************************************************/ + +#ifndef GNC_TRANS_PROPS_HPP +#define GNC_TRANS_PROPS_HPP + +extern "C" { +#include +#if PLATFORM(WINDOWS) +#include +#endif + +#include +} + +#include +#include + +/** Enumeration for column types. These are the different types of + * columns that can exist in a CSV/Fixed-Width file. There should be + * no two columns with the same type except for the GncTransPropType::NONE + * type. */ +enum class GncTransPropType { + NONE, + DATE, + DESCRIPTION, + NOTES, + TRANS_PROPS = NOTES, + + // num is strictly speaking a trans prop and not a split prop + // however due to the num/action swap user option, it can only be + // set while creating splits... + NUM, + + ACCOUNT, + DEPOSIT, + WITHDRAWAL, + BALANCE, + MEMO, + OACCOUNT, + OMEMO, + SPLIT_PROPS = OMEMO +}; + +/** Maps all column types to a string representation. + * The actual definition is in gnc-csv-imp-trans.cpp. + * Attention: that definition should be adjusted for any + * changes to enum class GncTransPropType ! */ +extern std::map gnc_csv_col_type_strs; + +time64 parse_date (const std::string &date_str, int format); + +struct GncPreTrans +{ +public: + void set_property (GncTransPropType prop_type, const std::string& prop_value_str, int date_format = 0); + std::string verify_essentials (void); + Transaction *create_trans (QofBook* book, gnc_commodity* currency); + +private: + boost::optional m_date; + boost::optional m_desc; + boost::optional m_notes; + bool created = false; +}; + +struct GncPreSplit +{ +public: + void set_property (GncTransPropType prop_type, const std::string& prop_value_str, int currency_format = 0); + std::string verify_essentials (void); + boost::optional create_split(Transaction* trans); + + Account* get_account () { if (m_account) return *m_account; else return nullptr; } + void set_account (Account* acct) { if (acct) m_account = acct; else m_account = boost::none; } + +private: + boost::optional m_account; + boost::optional m_deposit; + boost::optional m_withdrawal; + boost::optional m_balance; + boost::optional m_memo; + boost::optional m_oaccount; + boost::optional m_omemo; + + // Strictly speaking num is a transaction property + // However due to the option to swap num and action fields + // This can only be set when splits are created + boost::optional m_num; + bool created = false; +}; + + + +#endif diff --git a/src/import-export/csv-imp/gnc-tx-import.cpp b/src/import-export/csv-imp/gnc-tx-import.cpp index de0428e79e..248f245a42 100644 --- a/src/import-export/csv-imp/gnc-tx-import.cpp +++ b/src/import-export/csv-imp/gnc-tx-import.cpp @@ -30,26 +30,13 @@ extern "C" { #endif #include - -#include "gnc-csv-account-map.h" -#include "gnc-ui-util.h" -#include "engine-helpers.h" - -#include - -#include -#include -#include -#include -#include -#include } -#include #include #include #include "gnc-tx-import.hpp" +#include "gnc-trans-props.hpp" #include "gnc-csv-tokenizer.hpp" #include "gnc-fw-tokenizer.hpp" @@ -63,131 +50,12 @@ G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT; // N_("m-d") // }; // -/* Regular expressions used to parse dates per date format */ -const char* date_regex[] = { - "(?:" // either y-m-d - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)" - "|" // or CCYYMMDD - "(?[0-9]{4})" - "(?[0-9]{2})" - "(?[0-9]{2})" - ")", - - "(?:" // either d-m-y - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)" - "|" // or DDMMCCYY - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]{4})" - ")", - - "(?:" // either m-d-y - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)" - "|" // or MMDDCCYY - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]{4})" - ")", - - "(?:" // either d-m(-y) - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)(?:[-/.' ]+" - "(?[0-9]+))?" - "|" // or DDMM(CCYY) - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]+)?" - ")", - - "(?:" // either m-d(-y) - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)(?:[-/.' ]+" - "(?[0-9]+))?" - "|" // or MMDD(CCYY) - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]+)?" - ")", -}; //const int num_currency_formats = 3; //const gchar* currency_format_user[] = {N_("Locale"), // N_("Period: 123,456.78"), // N_("Comma: 123.456,78") // }; // -/* This map contains a set of strings representing the different column types. */ -std::map gnc_csv_col_type_strs = { - { GncTransPropType::NONE, N_("None") }, - { GncTransPropType::DATE, N_("Date") }, - { GncTransPropType::NUM, N_("Num") }, - { GncTransPropType::DESCRIPTION, N_("Description") }, - { GncTransPropType::NOTES, N_("Notes") }, - { GncTransPropType::ACCOUNT, N_("Account") }, - { GncTransPropType::DEPOSIT, N_("Deposit") }, - { GncTransPropType::WITHDRAWAL, N_("Withdrawal") }, - { GncTransPropType::BALANCE, N_("Balance") }, - { GncTransPropType::MEMO, N_("Memo") }, - { GncTransPropType::OACCOUNT, N_("Other Account") }, - { GncTransPropType::OMEMO, N_("Other Memo") } -}; - -/** Parses a string into a date, given a format. This function - * requires only knowing the order in which the year, month and day - * appear. For example, 01-02-2003 will be parsed the same way as - * 01/02/2003. - * @param date_str The string containing a date being parsed - * @param format An index specifying a format in date_format_user - * @exception std::invalid_argument if the string can't be parsed into a date. - * @return The parsed value of date_str on success, throws on failure - */ -time64 parse_date (const std::string &date_str, int format) -{ - boost::regex r(date_regex[format]); - boost::smatch what; - if(!boost::regex_search(date_str, what, r)) - throw std::invalid_argument ("String doesn't appear to be formatted as a date."); // regex didn't find a match - - // Attention: different behavior from 2.6.x series ! - // If date format without year was selected, the match - // should NOT have found a year. - if ((format >= 3) && (what.length("YEAR") != 0)) - throw std::invalid_argument ("String appears to contain a year while the selected format forbids this."); - - auto day = std::stoi (what.str("DAY")); - auto month = std::stoi (what.str("MONTH")); - - int year; - if (format < 3) - { - /* The input dates have a year, so use that one */ - year = std::stoi (what.str("YEAR")); - - /* Handle two-digit years. */ - if (year < 100) - { - /* We allow two-digit years in the range 1969 - 2068. */ - if (year < 69) - year += 2000; - else - year += 1900; - } - } - else - { - /* The input dates don't have a year, so work with today's year. - */ - gnc_timespec2dmy(timespec_now(), nullptr, nullptr, &year); - } - - auto ts = gnc_dmy2timespec_neutral(day, month, year); - return ts.tv_sec; -} /** Constructor for GncTxImport. * @return Pointer to a new GncCSvParseData @@ -337,368 +205,73 @@ void GncTxImport::parse (bool guessColTypes) } -/** Convert str into a time64 using the user-specified (import) date format. - * @param str The string to be parsed - * @param date_format The date format to use. - * @return a pointer to a time64 on success, nullptr on failure - */ -static time64* convert_date_col_str (const std::string &str, int date_format) -{ - try - { - auto parsed_date = parse_date (str.c_str(), date_format); - auto mydate = new time64; - *mydate = parsed_date; - return mydate; - } - catch (std::invalid_argument) - { - return nullptr; - } -} - - -/** Convert str into a gnc_numeric using the user-specified (import) currency format. - * @param str The string to be parsed - * @param currency_format The currency format to use. - * @return a pointer to a gnc_numeric on success, nullptr on failure - */ -static gnc_numeric* convert_amount_col_str (const std::string &str, int currency_format) -{ - /* If a cell is empty or just spaces return 0 as amount */ - if(!boost::regex_search(str, boost::regex("[0-9]"))) - return nullptr; - - auto expr = boost::make_u32regex("[[:Sc:]]"); - std::string str_no_symbols = boost::u32regex_replace(str, expr, ""); - - /* Convert based on user chosen currency format */ - gnc_numeric val; - char *endptr; - switch (currency_format) - { - case 0: - /* Currency locale */ - if (!(xaccParseAmount (str_no_symbols.c_str(), TRUE, &val, &endptr))) - return nullptr; - break; - case 1: - /* Currency decimal period */ - if (!(xaccParseAmountExtended (str_no_symbols.c_str(), TRUE, '-', '.', ',', "\003\003", "$+", &val, &endptr))) - return nullptr; - break; - case 2: - /* Currency decimal comma */ - if (!(xaccParseAmountExtended (str_no_symbols.c_str(), TRUE, '-', ',', '.', "\003\003", "$+", &val, &endptr))) - return nullptr; - break; - } - - auto amount = new gnc_numeric; - *amount = val; - return amount; -} - -/* Define a class hierarchy to temporarily store transaction/split properties - * found in one import line. There is a generic parent class and an implementation - * template class. This template class is further specialized for each data type - * we support (currently time64, Account, string and gnc_numeric). - */ -struct GncTransProperty -{ - virtual ~GncTransProperty() - //Remove pure designation. - {} - bool m_valid = false; -}; - -template -struct GncTransPropImpl -: public GncTransProperty -{ -public: - ~GncTransPropImpl(){}; - GncTransPropImpl(const std::string& val, int fmt) - { - m_valid = false; - }; - - static GncTransProperty* make_new(const std::string& val,int fmt = 0) - { return nullptr; } - - T m_value; -}; - -template<> -struct GncTransPropImpl -: public GncTransProperty -{ - GncTransPropImpl(const std::string& val, int fmt) - { - m_value = convert_date_col_str (val, fmt); - m_valid = (m_value != nullptr); - } - ~GncTransPropImpl() - { if (m_value) delete m_value; } - - static std::shared_ptr make_new(const std::string& val,int fmt) - { return std::shared_ptr(new GncTransPropImpl(val, fmt)); } - - time64* m_value; -}; - - -template<> -struct GncTransPropImpl -: public GncTransProperty -{ - GncTransPropImpl(const std::string& val, int fmt = 0) - { - m_value = new std::string(val); - m_valid = (m_value != nullptr); - } - ~GncTransPropImpl() - { if (m_value) delete m_value; } - - static std::shared_ptr make_new(const std::string& val,int fmt = 0) - { return std::shared_ptr(new GncTransPropImpl(val)); } /* Note fmt is not used for strings */ - - std::string* m_value; -}; - -template<> -struct GncTransPropImpl -: public GncTransProperty -{ - GncTransPropImpl(const std::string& val, int fmt = 0) - { - m_value = gnc_csv_account_map_search (val.c_str()); - m_valid = (m_value != nullptr); - } - GncTransPropImpl(Account* val) - { m_value = val; } - ~GncTransPropImpl(){}; - - static std::shared_ptr make_new(const std::string& val,int fmt = 0) - { return std::shared_ptr(new GncTransPropImpl(val)); } /* Note fmt is not used in for accounts */ - - Account * m_value; -}; - -template<> -struct GncTransPropImpl -: public GncTransProperty -{ - GncTransPropImpl(const std::string& val, int fmt) - { - m_value = convert_amount_col_str (val, fmt); - m_valid = (m_value != nullptr); - } - ~GncTransPropImpl() - { if (m_value) delete m_value; } - - static std::shared_ptr make_new(const std::string& val,int fmt) - { return std::shared_ptr(new GncTransPropImpl(val, fmt)); } - - gnc_numeric * m_value; -}; - -/** Adds a split to a transaction. - * @param trans The transaction to add a split to - * @param account The account used for the split - * @param book The book where the split should be stored - * @param amount The amount of the split - */ -static void trans_add_split (Transaction* trans, Account* account, QofBook* book, - gnc_numeric amount, const std::string& num, const std::string& memo) -{ - auto split = xaccMallocSplit (book); - xaccSplitSetAccount (split, account); - xaccSplitSetParent (split, trans); - xaccSplitSetAmount (split, amount); - xaccSplitSetValue (split, amount); - if (!memo.empty()) - xaccSplitSetMemo (split, memo.c_str()); - /* set tran-num and/or split-action per book option - * note this function does nothing if num is NULL also */ - if (!num.empty()) - gnc_set_num_action (trans, split, num.c_str(), NULL); -} - - -/* Shorthand aliases for the container to keep track of property types (a map) - * and its iterator (a pair) - */ -using prop_pair_t = std::pair>; - -/** Tests a TransPropertyList for having enough essential properties. +/** Checks whether the parsed line contains all essential properties. * Essential properties are * - "Date" * - at least one of "Balance", "Deposit", or "Withdrawal" * - "Account" * Note account isn't checked for here as this has been done before - * @param list The list we are checking - * @param error Contains an error message on failure - * @return true if there are enough essentials; false otherwise + * @param parse_line The line we are checking + * @exception std::invalid_argument in an essential property is missing */ -static bool trans_properties_verify_essentials (prop_map_t& trans_props, gchar** error) +static void trans_properties_verify_essentials (parse_line_t& orig_line) { - /* Make sure this is a transaction with all the columns we need. */ - bool have_date = (trans_props.find (GncTransPropType::DATE) != trans_props.end()); - bool have_amount = ((trans_props.find (GncTransPropType::DEPOSIT) != trans_props.end()) || - (trans_props.find (GncTransPropType::WITHDRAWAL) != trans_props.end()) || - (trans_props.find (GncTransPropType::BALANCE) != trans_props.end())); + std::string error_message; + std::shared_ptr trans_props; + std::shared_ptr split_props; - std::string error_message {""}; - if (!have_date) - error_message += N_("No date column."); - if (!have_amount) + std::tie(std::ignore, error_message, trans_props, split_props) = orig_line; + + auto trans_error = trans_props->verify_essentials(); + auto split_error = split_props->verify_essentials(); + + error_message.clear(); + if (!trans_error.empty()) { - if (!have_date) + error_message = trans_error; + if (!split_error.empty()) error_message += "\n"; - error_message += N_("No balance, deposit, or withdrawal column."); } - if (!have_date || !have_amount) - *error = g_strdup (error_message.c_str()); + if (!split_error.empty()) + error_message += split_error; - return have_amount && have_date; + if (!error_message.empty()) + throw std::invalid_argument(error_message); } - /** Create a Transaction from a map of transaction properties. * Note: this function assumes all properties in the map have been verified * to be valid. No further checks are performed here other than that * the required properties are in the map - * @param transprops The map of transaction properties - * @param error Contains an error on failure + * @param orig_line The current line being parsed * @return On success, a GncCsvTransLine; on failure, the trans pointer is NULL */ -static GncCsvTransLine* trans_properties_to_trans (prop_map_t& trans_props, gchar** error) +static GncCsvTransLine* trans_properties_to_trans (parse_line_t& orig_line) { - - if (!trans_properties_verify_essentials(trans_props, error)) - return NULL; - - auto property = trans_props.find (GncTransPropType::ACCOUNT)->second; - auto account = dynamic_cast*>(property.get())->m_value; - - GncCsvTransLine* trans_line = g_new (GncCsvTransLine, 1); - - /* The balance is 0 by default. */ - trans_line->balance_set = false; - trans_line->balance = double_to_gnc_numeric (0.0, xaccAccountGetCommoditySCU (account), - GNC_HOW_RND_ROUND_HALF_UP); + std::string error_message; + std::shared_ptr trans_props; + std::shared_ptr split_props; + std::tie(std::ignore, error_message, trans_props, split_props) = orig_line; + auto account = split_props->get_account(); QofBook* book = gnc_account_get_book (account); gnc_commodity* currency = xaccAccountGetCommodity (account); - trans_line->trans = xaccMallocTransaction (book); - xaccTransBeginEdit (trans_line->trans); - xaccTransSetCurrency (trans_line->trans, currency); - /* Go through each of the properties and edit the transaction accordingly. */ - std::string num; - std::string memo; - std::string omemo; - Account *oaccount = NULL; - bool amount_set = false; - gnc_numeric amount = trans_line->balance; + auto trans = trans_props->create_trans (book, currency); - for (auto prop_pair : trans_props) + if (!trans) + return nullptr; + + GncCsvTransLine* trans_line = g_new (GncCsvTransLine, 1); + trans_line->balance_set = false; + trans_line->balance = gnc_numeric_zero(); + + auto balance = split_props->create_split(trans); + if (balance) { - auto type = prop_pair.first; - auto prop = prop_pair.second; - switch (type) - { - case GncTransPropType::DATE: - { - auto transdate = dynamic_cast*>(prop.get())->m_value; - xaccTransSetDatePostedSecsNormalized (trans_line->trans, *transdate); - } - break; - - case GncTransPropType::DESCRIPTION: - { - auto propstring = dynamic_cast*>(prop.get())->m_value; - xaccTransSetDescription (trans_line->trans, propstring->c_str()); - } - break; - - case GncTransPropType::NOTES: - { - auto propstring = dynamic_cast*>(prop.get())->m_value; - xaccTransSetNotes (trans_line->trans, propstring->c_str()); - } - break; - - case GncTransPropType::OACCOUNT: - oaccount = dynamic_cast*>(prop.get())->m_value; - break; - - case GncTransPropType::MEMO: - memo = *dynamic_cast*>(prop.get())->m_value; - break; - - case GncTransPropType::OMEMO: - omemo = *dynamic_cast*>(prop.get())->m_value; - break; - - case GncTransPropType::NUM: - /* the 'num' is saved and passed to 'trans_add_split' below where - * 'gnc_set_num_action' is used to set tran-num and/or split-action - * per book option */ - num = *dynamic_cast*>(prop.get())->m_value; - break; - - case GncTransPropType::DEPOSIT: /* Add deposits to the existing amount. */ - { - auto propval = dynamic_cast*>(prop.get())->m_value; - amount = gnc_numeric_add (*propval, - amount, - xaccAccountGetCommoditySCU (account), - GNC_HOW_RND_ROUND_HALF_UP); - amount_set = true; - /* We will use the "Deposit" and "Withdrawal" columns in preference to "Balance". */ - trans_line->balance_set = false; - } - break; - - case GncTransPropType::WITHDRAWAL: /* Withdrawals are just negative deposits. */ - { - auto propval = dynamic_cast*>(prop.get())->m_value; - amount = gnc_numeric_add (gnc_numeric_neg(*propval), - amount, - xaccAccountGetCommoditySCU (account), - GNC_HOW_RND_ROUND_HALF_UP); - amount_set = true; - /* We will use the "Deposit" and "Withdrawal" columns in preference to "Balance". */ - trans_line->balance_set = false; - } - break; - - case GncTransPropType::BALANCE: /* The balance gets stored in a separate field in trans_line. */ - /* We will use the "Deposit" and "Withdrawal" columns in preference to "Balance". */ - if (!amount_set) - { - auto propval = dynamic_cast*>(prop.get())->m_value; - /* This gets put into the actual transaction at the end of gnc_csv_parse_to_trans. */ - trans_line->balance = *propval; - trans_line->balance_set = true; - } - break; - default: - break; - } + trans_line->balance_set = true; + trans_line->balance = *balance; } - /* Add a split with the cumulative amount value. */ - trans_add_split (trans_line->trans, account, book, amount, num, memo); - - if (oaccount) - /* Note: the current importer assumes at most 2 splits. This means the second split amount - * will be the negative of the the first split amount. We also only set the num field once, - * for the first split. - */ - trans_add_split (trans_line->trans, oaccount, book, gnc_numeric_neg(amount), "", omemo); - return trans_line; } @@ -753,8 +326,15 @@ void GncTxImport::adjust_balances (Account *account) } -void GncTxImport::parse_line_to_trans (StrVec& line, prop_map_t& trans_props) +void GncTxImport::parse_line_to_trans (parse_line_t& orig_line) { + StrVec line; + std::string error_message; + std::shared_ptr trans_props; + std::shared_ptr split_props; + std::tie(line, error_message, trans_props, split_props) = orig_line; + error_message.clear(); + /* Convert this import line into a map of transaction/split properties. */ auto col_types_it = column_types.cbegin(); auto line_it = line.cbegin(); @@ -763,49 +343,68 @@ void GncTxImport::parse_line_to_trans (StrVec& line, prop_map_t& trans_props) line_it != line.cend(); ++col_types_it, ++line_it) { - std::shared_ptr property; - switch (*col_types_it) + try { - case GncTransPropType::DATE: - property = GncTransPropImpl::make_new (*line_it, date_format); - break; - - case GncTransPropType::DESCRIPTION: - case GncTransPropType::NOTES: - case GncTransPropType::MEMO: - case GncTransPropType::OMEMO: - case GncTransPropType::NUM: - property = GncTransPropImpl::make_new (*line_it); - break; - - case GncTransPropType::ACCOUNT: - case GncTransPropType::OACCOUNT: - property = GncTransPropImpl::make_new (*line_it); - break; - - case GncTransPropType::BALANCE: - case GncTransPropType::DEPOSIT: - case GncTransPropType::WITHDRAWAL: - property = GncTransPropImpl::make_new (*line_it, currency_format); - break; - - default: + if (*col_types_it == GncTransPropType::NONE) continue; /* We do nothing with "None"-type columns. */ - break; + else if (*col_types_it <= GncTransPropType::TRANS_PROPS) + trans_props->set_property(*col_types_it, *line_it, date_format); + else + split_props->set_property(*col_types_it, *line_it, currency_format); } - - if (property->m_valid) - trans_props.insert(prop_pair_t(*col_types_it, property)); - else + catch (const std::invalid_argument&) { - std::string error_message {_(gnc_csv_col_type_strs[*col_types_it])}; + parse_errors = true; + error_message += _(gnc_csv_col_type_strs[*col_types_it]); error_message += _(" column could not be understood."); - throw std::invalid_argument (error_message); + error_message += "\n"; } } + if (!error_message.empty()) + throw std::invalid_argument(error_message); + + // Add an ACCOUNT property with the default account if no account column was set by the user + auto line_acct = split_props->get_account(); + if (!line_acct) + { + if (home_account) + split_props->set_account(home_account); + else + { + // Oops - the user didn't select an Account column *and* we didn't get a default value either! + // Note if you get here this suggests a bug in the code! + parse_errors = true; + error_message = _("No account column selected and no default account specified either.\n" + "This should never happen. Please report this as a bug."); + throw std::invalid_argument(error_message); + } + } + + /* If column parsing was successful, convert trans properties into a trans line. */ + try + { + trans_properties_verify_essentials (orig_line); + + /* If all went well, add this transaction to the list. */ + /* We want to keep the transactions sorted by date in case we have + * to calculate the transaction's amount based on the user provided balances. + * The multimap should deal with this for us. */ + auto trans_line = trans_properties_to_trans (orig_line); + if (trans_line) + { + auto trans_date = xaccTransGetDate (trans_line->trans); + transactions.insert (std::pair(trans_date,trans_line)); + } + } + catch (const std::invalid_argument& e) + { + parse_errors = true; + error_message = e.what(); + } } + /** Creates a list of transactions from parsed data. Transactions that * could be created from rows are placed in transactions; * rows that fail are placed in error_lines. (Note: there @@ -843,14 +442,13 @@ int GncTxImport::parse_to_trans (Account* account, else std::advance(orig_lines_max, end_row); - Account *home_account = NULL; + home_account = account; auto odd_line = false; parse_errors = false; for (orig_lines_it, odd_line; orig_lines_it != orig_lines_max; ++orig_lines_it, odd_line = !odd_line) { - prop_map_t trans_props; auto orig_line = *orig_lines_it; /* Skip current line if: @@ -865,60 +463,17 @@ int GncTxImport::parse_to_trans (Account* account, try { - parse_line_to_trans (std::get<0>(orig_line), trans_props); + parse_line_to_trans (orig_line); } - catch (const std::invalid_argument& e) + catch (const std::invalid_argument&) { - parse_errors = true; - std::get<1>(orig_line) = e.what(); continue; } - - // Add an ACCOUNT property with the default account if no account column was set by the user - auto acct_prop_it = trans_props.find (GncTransPropType::ACCOUNT); - if (acct_prop_it != trans_props.end()) - home_account = dynamic_cast*>(acct_prop_it->second.get())->m_value; - else - { - // If there is no ACCOUNT property by now, try to use the default account passed in - if (account) - { - auto property = std::shared_ptr(new GncTransPropImpl(account)); - trans_props.insert(prop_pair_t(GncTransPropType::ACCOUNT, property)); - home_account = account; - } - else - { - // Oops - the user didn't select an Account column *and* we didn't get a default value either! - // Note if you get here this suggests a bug in the code! - parse_errors = true; - std::get<1>(orig_line) = _("No account column selected and no default account specified either."); - continue; - } - } - - /* If column parsing was successful, convert trans properties into a trans line. */ - gchar *error_message = NULL; - auto trans_line = trans_properties_to_trans (trans_props, &error_message); - if (trans_line == NULL) - { - parse_errors = true; - std::get<1>(orig_line) = error_message; - g_free (error_message); - continue; - } - - /* If all went well, add this transaction to the list. */ - /* We want to keep the transactions sorted by date in case we have - * to calculate the transaction's amount based on the user provided balances. - * The multimap should deal with this for us. */ - auto trans_date = xaccTransGetDate (trans_line->trans); - transactions.insert (std::pair(trans_date,trans_line)); } if (std::find(column_types.begin(),column_types.end(), GncTransPropType::BALANCE) != column_types.end()) // This is only used if we have one home account - adjust_balances (account ? account : home_account); + adjust_balances (home_account); return 0; } diff --git a/src/import-export/csv-imp/gnc-tx-import.hpp b/src/import-export/csv-imp/gnc-tx-import.hpp index 9ea3d67620..f9ab224073 100644 --- a/src/import-export/csv-imp/gnc-tx-import.hpp +++ b/src/import-export/csv-imp/gnc-tx-import.hpp @@ -39,36 +39,11 @@ extern "C" { #include #include #include -#include #include "gnc-tokenizer.hpp" +#include "gnc-trans-props.hpp" -/** Enumeration for column types. These are the different types of - * columns that can exist in a CSV/Fixed-Width file. There should be - * no two columns with the same type except for the GncTransPropType::NONE - * type. */ -enum class GncTransPropType { - NONE, - DATE, - NUM, - DESCRIPTION, - NOTES, - ACCOUNT, - DEPOSIT, - WITHDRAWAL, - BALANCE, - MEMO, - OACCOUNT, - OMEMO -}; - -/** Maps all column types to a string representation. - * The actual definition is in gnc-tx-import.cpp. - * Attention: that definition should be adjusted for any - * changes to enum class GncTransPropType ! */ -extern std::map gnc_csv_col_type_strs; - /* TODO We now sort transactions by date, not line number, so we * should probably get rid of this struct and uses of it. */ @@ -94,25 +69,6 @@ extern const gchar* currency_format_user[]; extern const int num_date_formats; extern const gchar* date_format_user[]; -struct GncPreTrans -{ - boost::optional m_date; - boost::optional m_num; - boost::optional m_desc; - boost::optional m_notes; -}; - -struct GncPreSplit -{ - boost::optional m_account; - boost::optional m_deposit; - boost::optional m_withdrawal; - boost::optional m_balance; - boost::optional m_memo; - boost::optional m_oaccount; - boost::optional m_omemo; -}; - /** Tuple to hold * - a tokenized line of input * - an optional error string @@ -123,9 +79,6 @@ using parse_line_t = std::tuple, std::shared_ptr>; -struct GncTransProperty; -using prop_map_t = std::map>; - /** The actual TxImport class * It's intended to use in the following sequence of actions: * - set a file format @@ -165,12 +118,12 @@ public: bool parse_errors; /**< Indicates whether the last parse_to_trans run had any errors */ private: - void parse_line_to_trans (StrVec& line, prop_map_t& trans_props); + void parse_line_to_trans (parse_line_t& orig_line); void adjust_balances (Account *account); GncImpFileFormat file_fmt = GncImpFileFormat::UNKNOWN; + Account *home_account = NULL; }; -time64 parse_date (const std::string &date_str, int format); #endif