From 9af57849ba1bc6e954f6b5aae61ec85e4c09a0e2 Mon Sep 17 00:00:00 2001 From: Geert Janssens Date: Mon, 1 May 2017 20:19:49 +0200 Subject: [PATCH] Use GncDate in csv importer This removes all date specific functionality from the importer --- .../csv-imp/assistant-csv-trans-import.cpp | 8 +- src/import-export/csv-imp/gnc-trans-props.cpp | 125 ++------------- src/import-export/csv-imp/gnc-trans-props.hpp | 8 +- src/import-export/csv-imp/gnc-tx-import.cpp | 10 +- src/import-export/csv-imp/gnc-tx-import.hpp | 4 - .../csv-imp/test/test-tx-import.cpp | 147 ------------------ 6 files changed, 19 insertions(+), 283 deletions(-) diff --git a/src/import-export/csv-imp/assistant-csv-trans-import.cpp b/src/import-export/csv-imp/assistant-csv-trans-import.cpp index 05219dab45..3cd6c60e13 100644 --- a/src/import-export/csv-imp/assistant-csv-trans-import.cpp +++ b/src/import-export/csv-imp/assistant-csv-trans-import.cpp @@ -521,10 +521,8 @@ CsvImpTransAssist::CsvImpTransAssist () /* Add in the date format combo box and hook it up to an event handler. */ date_format_combo = GTK_COMBO_BOX_TEXT(gtk_combo_box_text_new()); - for (int i = 0; i < num_date_formats; i++) - { - gtk_combo_box_text_append_text (date_format_combo, _(date_format_user[i])); - } + for (auto& date_fmt : GncDate::c_formats) + gtk_combo_box_text_append_text (date_format_combo, _(date_fmt.m_fmt.c_str())); gtk_combo_box_set_active (GTK_COMBO_BOX(date_format_combo), 0); g_signal_connect (G_OBJECT(date_format_combo), "changed", G_CALLBACK(csv_tximp_preview_date_fmt_sel_cb), this); @@ -1560,7 +1558,7 @@ CsvImpTransAssist::preview_refresh () gtk_toggle_button_set_active (GTK_TOGGLE_BUTTON(fixed_button), (tx_imp->file_format() != GncImpFileFormat::CSV)); - // This section deals with the combo's and character encoding + // Set Date & Currency Format and Character encoding gtk_combo_box_set_active (GTK_COMBO_BOX(date_format_combo), tx_imp->date_format()); gtk_combo_box_set_active (GTK_COMBO_BOX(currency_format_combo), diff --git a/src/import-export/csv-imp/gnc-trans-props.cpp b/src/import-export/csv-imp/gnc-trans-props.cpp index c537d5430e..363a6299d0 100644 --- a/src/import-export/csv-imp/gnc-trans-props.cpp +++ b/src/import-export/csv-imp/gnc-trans-props.cpp @@ -93,115 +93,6 @@ GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split) return GncTransPropType::NONE; } -/* Regular expressions used to parse dates per date format */ -const char* date_regex[] = { - "(?:" // either y-m-d - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)" - "|" // or CCYYMMDD - "(?[0-9]{4})" - "(?[0-9]{2})" - "(?[0-9]{2})" - ")", - - "(?:" // either d-m-y - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)" - "|" // or DDMMCCYY - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]{4})" - ")", - - "(?:" // either m-d-y - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)" - "|" // or MMDDCCYY - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]{4})" - ")", - - "(?:" // either d-m(-y) - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)(?:[-/.' ]+" - "(?[0-9]+))?" - "|" // or DDMM(CCYY) - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]+)?" - ")", - - "(?:" // either m-d(-y) - "(?[0-9]+)[-/.' ]+" - "(?[0-9]+)(?:[-/.' ]+" - "(?[0-9]+))?" - "|" // or MMDD(CCYY) - "(?[0-9]{2})" - "(?[0-9]{2})" - "(?[0-9]+)?" - ")", -}; - -/** Parses a string into a date, given a format. This function - * requires only knowing the order in which the year, month and day - * appear. For example, 01-02-2003 will be parsed the same way as - * 01/02/2003. - * @param date_str The string containing a date being parsed - * @param format An index specifying a format in date_format_user - * @exception std::invalid_argument if the string can't be parsed into a date. - * @return The parsed value of date_str on success, throws on failure - */ - -time64 parse_date (const std::string &date_str, int format) -{ - boost::regex r(date_regex[format]); - boost::smatch what; - if(!boost::regex_search(date_str, what, r)) - throw std::invalid_argument (_("Value can't be parsed into a date using the selected date format.")); // regex didn't find a match - - // Attention: different behavior from 2.6.x series ! - // If date format without year was selected, the match - // should NOT have found a year. - if ((format >= 3) && (what.length("YEAR") != 0)) - throw std::invalid_argument (_("Value appears to contain a year while the selected format forbids this.")); - - auto day = std::stoi (what.str("DAY")); - auto month = std::stoi (what.str("MONTH")); - - int year; - if (format < 3) - { - /* The input dates have a year, so use that one */ - year = std::stoi (what.str("YEAR")); - - /* Handle two-digit years. */ - if (year < 100) - { - /* We allow two-digit years in the range 1969 - 2068. */ - if (year < 69) - year += 2000; - else - year += 1900; - } - } - else - { - /* The input dates don't have a year, so work with today's year. - */ - gnc_timespec2dmy(timespec_now(), nullptr, nullptr, &year); - } - - auto ts = gnc_dmy2timespec_neutral(day, month, year); - if (ts.tv_sec == INT64_MAX) - throw std::invalid_argument (_("Value can't be parsed into a date using the selected date format.")); - - return ts.tv_sec; -} - /** Convert str into a GncRational using the user-specified (import) currency format. * @param str The string to be parsed @@ -317,7 +208,7 @@ void GncPreTrans::set (GncTransPropType prop_type, const std::string& value) case GncTransPropType::DATE: m_date = boost::none; - m_date = parse_date (value, m_date_format); // Throws if parsing fails + m_date = GncDate(value, GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails break; case GncTransPropType::NUM: @@ -417,7 +308,8 @@ Transaction* GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency) auto trans = xaccMallocTransaction (book); xaccTransBeginEdit (trans); xaccTransSetCurrency (trans, m_commodity ? *m_commodity : currency); - xaccTransSetDatePostedSecsNormalized (trans, *m_date); + xaccTransSetDatePostedSecsNormalized (trans, + static_cast(GncDateTime(*m_date, DayPart::neutral))); if (m_num) xaccTransSetNum (trans, m_num->c_str()); @@ -558,13 +450,15 @@ void GncPreSplit::set (GncTransPropType prop_type, const std::string& value) case GncTransPropType::REC_DATE: m_rec_date = boost::none; if (!value.empty()) - m_rec_date = parse_date (value, m_date_format); // Throws if parsing fails + m_rec_date = GncDate (value, + GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails break; case GncTransPropType::TREC_DATE: m_trec_date = boost::none; if (!value.empty()) - m_trec_date = parse_date (value, m_date_format); // Throws if parsing fails + m_trec_date = GncDate (value, + GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails break; default: @@ -641,7 +535,7 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am const boost::optional& action, const boost::optional& memo, const boost::optional& rec_state, - const boost::optional rec_date, + const boost::optional& rec_date, const boost::optional price) { QofBook* book = xaccTransGetBook (trans); @@ -691,7 +585,8 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am if (rec_state && *rec_state != 'n') xaccSplitSetReconcile (split, *rec_state); if (rec_state && *rec_state == YREC && rec_date) - xaccSplitSetDateReconciledSecs (split, *rec_date); + xaccSplitSetDateReconciledSecs (split, + static_cast(GncDateTime(*rec_date, DayPart::neutral))); } diff --git a/src/import-export/csv-imp/gnc-trans-props.hpp b/src/import-export/csv-imp/gnc-trans-props.hpp index 8188206320..47c23bad7d 100644 --- a/src/import-export/csv-imp/gnc-trans-props.hpp +++ b/src/import-export/csv-imp/gnc-trans-props.hpp @@ -41,6 +41,7 @@ extern "C" { #include #include #include +#include #include /** Enumeration for column types. These are the different types of @@ -103,7 +104,6 @@ private: GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split); -time64 parse_date (const std::string &date_str, int format); gnc_commodity* parse_commodity (const std::string& comm_str); GncNumeric parse_amount (const std::string &str, int currency_format); @@ -139,7 +139,7 @@ public: private: int m_date_format; boost::optional m_differ; - boost::optional m_date; + boost::optional m_date; boost::optional m_num; boost::optional m_desc; boost::optional m_notes; @@ -176,12 +176,12 @@ private: boost::optional m_price; boost::optional m_memo; boost::optional m_rec_state; - boost::optional m_rec_date; + boost::optional m_rec_date; boost::optional m_taction; boost::optional m_taccount; boost::optional m_tmemo; boost::optional m_trec_state; - boost::optional m_trec_date; + boost::optional m_trec_date; bool created = false; std::map m_errors; diff --git a/src/import-export/csv-imp/gnc-tx-import.cpp b/src/import-export/csv-imp/gnc-tx-import.cpp index 7dbaff82d0..daa85b4d98 100644 --- a/src/import-export/csv-imp/gnc-tx-import.cpp +++ b/src/import-export/csv-imp/gnc-tx-import.cpp @@ -43,14 +43,6 @@ extern "C" { G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT; -const int num_date_formats = 5; -const gchar* date_format_user[] = {N_("y-m-d"), - N_("d-m-y"), - N_("m-d-y"), - N_("d-m"), - N_("m-d") - }; - const int num_currency_formats = 3; const gchar* currency_format_user[] = {N_("Locale"), N_("Period: 123,456.78"), @@ -742,6 +734,8 @@ void GncTxImport::update_pre_trans_props (uint32_t row, uint32_t col, GncTransPr if ((prop_type == GncTransPropType::NONE) || (prop_type > GncTransPropType::TRANS_PROPS)) return; /* Only deal with transaction related properties. */ + /* Deliberately make a copy of the GncPreTrans. It may be the original one was shared + * with a previous line and should no longer be after the transprop is changed. */ auto trans_props = std::make_shared (*(std::get(m_parsed_lines[row])).get()); auto value = std::string(); diff --git a/src/import-export/csv-imp/gnc-tx-import.hpp b/src/import-export/csv-imp/gnc-tx-import.hpp index fbded6e968..4e7482c3c9 100644 --- a/src/import-export/csv-imp/gnc-tx-import.hpp +++ b/src/import-export/csv-imp/gnc-tx-import.hpp @@ -64,10 +64,6 @@ struct DraftTransaction extern const int num_currency_formats; extern const gchar* currency_format_user[]; -/* A set of date formats that the user sees. */ -extern const int num_date_formats; -extern const gchar* date_format_user[]; - /** An enum describing the columns found in a parse_line_t. Currently these are: * - a tokenized line of input * - an optional error string diff --git a/src/import-export/csv-imp/test/test-tx-import.cpp b/src/import-export/csv-imp/test/test-tx-import.cpp index 1df18a4518..76a43625fa 100644 --- a/src/import-export/csv-imp/test/test-tx-import.cpp +++ b/src/import-export/csv-imp/test/test-tx-import.cpp @@ -62,150 +62,3 @@ public: protected: std::unique_ptr tx_importer; }; - - -/* parse_date -time64 parse_date (const char* date_str, int format)// C: 14 in 7 SCM: 9 in 2 Local: 1:0:0 -*/ -TEST(GncTxImportTest, parse_date) -{ - time64 rawtime = gnc_time (nullptr); - struct tm *tm = gnc_gmtime (&rawtime); - int curr_year = tm->tm_year; - - - /* Note: tm_year = year - 1900 and tm_mon = 0-11 - * I'm writing the expected values as subtractions for easier - * comparison with the date string under test - */ - parse_date_data test_dates[] = - { - // supported combinations -/.' - { 0, "2013-08-01", 2013 - 1900, 8 - 1, 1}, - { 0, "2013-8-01", 2013 - 1900, 8 - 1, 1}, - { 0, "2013-08-1", 2013 - 1900, 8 - 1, 1}, - { 0, "2013-8-1", 2013 - 1900, 8 - 1, 1}, - { 0, "13-08-01", 2013 - 1900, 8 - 1, 1}, - { 0, "13-8-01", 2013 - 1900, 8 - 1, 1}, - { 0, "13-08-1", 2013 - 1900, 8 - 1, 1}, - { 0, "13-8-1", 2013 - 1900, 8 - 1, 1}, - { 0, "2009/11/04", 2009 - 1900, 11 - 1, 4}, - { 0, "1985.3.12", 1985 - 1900, 3 - 1, 12}, - { 0, "3'6'8", 2003 - 1900, 6 - 1, 8}, - { 0, "20130801", 2013 - 1900, 8 - 1, 1}, - { 1, "01-08-2013", 2013 - 1900, 8 - 1, 1}, - { 1, "01-8-2013", 2013 - 1900, 8 - 1, 1}, - { 1, "1-08-2013", 2013 - 1900, 8 - 1, 1}, - { 1, "1-8-2013", 2013 - 1900, 8 - 1, 1}, - { 1, "01-08-13", 2013 - 1900, 8 - 1, 1}, - { 1, "01-8-13", 2013 - 1900, 8 - 1, 1}, - { 1, "1-08-13", 2013 - 1900, 8 - 1, 1}, - { 1, "1-8-13", 2013 - 1900, 8 - 1, 1}, - { 1, "04/11/2009", 2009 - 1900, 11 - 1, 4}, - { 1, "12.3.1985", 1985 - 1900, 3 - 1, 12}, - { 1, "8'6'3", 2003 - 1900, 6 - 1, 8}, - { 1, "01082013", 2013 - 1900, 8 - 1, 1}, - { 2, "08-01-2013", 2013 - 1900, 8 - 1, 1}, - { 2, "8-01-2013", 2013 - 1900, 8 - 1, 1}, - { 2, "08-1-2013", 2013 - 1900, 8 - 1, 1}, - { 2, "8-1-2013", 2013 - 1900, 8 - 1, 1}, - { 2, "08-01-13", 2013 - 1900, 8 - 1, 1}, - { 2, "8-01-13", 2013 - 1900, 8 - 1, 1}, - { 2, "08-1-13", 2013 - 1900, 8 - 1, 1}, - { 2, "8-1-13", 2013 - 1900, 8 - 1, 1}, - { 2, "11/04/2009", 2009 - 1900, 11 - 1, 4}, - { 2, "3.12.1985", 1985 - 1900, 3 - 1, 12}, - { 2, "6'8'3", 2003 - 1900, 6 - 1, 8}, - { 2, "08012013", 2013 - 1900, 8 - 1, 1}, - { 3, "01-08", curr_year, 8 - 1, 1}, - { 3, "01-8", curr_year, 8 - 1, 1}, - { 3, "1-08", curr_year, 8 - 1, 1}, - { 3, "1-8", curr_year, 8 - 1, 1}, - { 3, "04/11", curr_year, 11 - 1, 4}, - { 3, "12.3", curr_year, 3 - 1, 12}, - { 3, "8'6", curr_year, 6 - 1, 8}, - { 3, "0108", curr_year, 8 - 1, 1}, - { 4, "08-01", curr_year, 8 - 1, 1}, - { 4, "8-01", curr_year, 8 - 1, 1}, - { 4, "08-1", curr_year, 8 - 1, 1}, - { 4, "8-1", curr_year, 8 - 1, 1}, - { 4, "11/04", curr_year, 11 - 1, 4}, - { 4, "3.12", curr_year, 3 - 1, 12}, - { 4, "6'8", curr_year, 6 - 1, 8}, - { 4, "0801", curr_year, 8 - 1, 1}, - - // ambiguous date formats - // current parser doesn't know how to disambiguate - // and hence refuses to parse - // can possibly improved with a smarter parser - { 0, "130801", -1, -1, -1}, - { 1, "010813", -1, -1, -1}, - { 2, "080113", -1, -1, -1}, - - // Combinations that don't make sense - // but can still be entered by a user - // Should ideally all result in refusal to parse... - { 0, "08-01", -1, -1, -1}, - { 0, "0801", -1, -1, -1}, - { 1, "01-08", -1, -1, -1}, - { 1, "0108", -1, -1, -1}, - { 2, "08-01", -1, -1, -1}, - { 2, "0801", -1, -1, -1}, - { 3, "01-08-2013", -1, -1, -1}, - { 3, "01-08-13", -1, -1, -1}, - { 3, "08-08-08", -1, -1, -1}, - { 3, "01082013", -1, -1, -1}, - { 3, "010813", -1, -1, -1}, - { 3, "20130108", -1, -1, -1}, - { 4, "08-01-2013", -1, -1, -1}, - { 4, "08-01-13", -1, -1, -1}, - { 4, "2013-08-01", -1, -1, -1}, - { 4, "09-08-01", -1, -1, -1}, - { 4, "08012013", -1, -1, -1}, - { 4, "080113", -1, -1, -1}, - { 4, "20130801", -1, -1, -1}, - - // Sentinel to mark the end of available tests - { 0, NULL, 0, 0, 0}, - - }; - int i = 0; - - gnc_tm_free(tm); - while (test_dates[i].date_str) - { - gboolean success = TRUE; - int got_year = 0, got_month = 0, got_day = 0; - - try - { - rawtime = parse_date (std::string(test_dates[i].date_str), test_dates[i].date_fmt); - tm = gnc_gmtime (&rawtime); - got_year = tm->tm_year; - got_month = tm->tm_mon; - got_day = tm->tm_mday; - gnc_tm_free(tm); - } - catch (std::invalid_argument) - { - got_year = got_month = got_day = -1; - } - - if ((got_year != test_dates[i].exp_year) || - (got_month != test_dates[i].exp_month) || - (got_day != test_dates[i].exp_day)) - { - g_error ("Parse_date failed for date '%s' and format '%d'.\n" - "Expected result: year %d, month %d, day %d\n" - "Obtained result: year %d, month %d, day %d", - test_dates[i].date_str, - test_dates[i].date_fmt, - test_dates[i].exp_year, - test_dates[i].exp_month, - test_dates[i].exp_day, - got_year, got_month, got_day); - } - - i++; - } -}