Use GncDate in csv importer

This removes all date specific functionality from the importer

src/import-export/csv-imp/assistant-csv-trans-import.cpp

@@ -521,10 +521,8 @@ CsvImpTransAssist::CsvImpTransAssist ()
 
         /* Add in the date format combo box and hook it up to an event handler. */
         date_format_combo = GTK_COMBO_BOX_TEXT(gtk_combo_box_text_new());
-        for (int i = 0; i < num_date_formats; i++)
+        for (auto& date_fmt : GncDate::c_formats)
-        {
+            gtk_combo_box_text_append_text (date_format_combo, _(date_fmt.m_fmt.c_str()));
-            gtk_combo_box_text_append_text (date_format_combo, _(date_format_user[i]));
-        }
         gtk_combo_box_set_active (GTK_COMBO_BOX(date_format_combo), 0);
         g_signal_connect (G_OBJECT(date_format_combo), "changed",
                          G_CALLBACK(csv_tximp_preview_date_fmt_sel_cb), this);
@@ -1560,7 +1558,7 @@ CsvImpTransAssist::preview_refresh ()
     gtk_toggle_button_set_active (GTK_TOGGLE_BUTTON(fixed_button),
             (tx_imp->file_format() != GncImpFileFormat::CSV));
 
-    // This section deals with the combo's and character encoding
+    // Set Date & Currency Format and Character encoding
     gtk_combo_box_set_active (GTK_COMBO_BOX(date_format_combo),
             tx_imp->date_format());
     gtk_combo_box_set_active (GTK_COMBO_BOX(currency_format_combo),

src/import-export/csv-imp/gnc-trans-props.cpp

@@ -93,115 +93,6 @@ GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split)
         return GncTransPropType::NONE;
 }
 
-/* Regular expressions used to parse dates per date format */
-const char* date_regex[] = {
-                             "(?:"                                   // either y-m-d
-                                 "(?<YEAR>[0-9]+)[-/.' ]+"
-                                 "(?<MONTH>[0-9]+)[-/.' ]+"
-                                 "(?<DAY>[0-9]+)"
-                             "|"                                     // or CCYYMMDD
-                                 "(?<YEAR>[0-9]{4})"
-                                 "(?<MONTH>[0-9]{2})"
-                                 "(?<DAY>[0-9]{2})"
-                             ")",
-
-                             "(?:"                                   // either d-m-y
-                                 "(?<DAY>[0-9]+)[-/.' ]+"
-                                 "(?<MONTH>[0-9]+)[-/.' ]+"
-                                 "(?<YEAR>[0-9]+)"
-                             "|"                                     // or DDMMCCYY
-                                 "(?<DAY>[0-9]{2})"
-                                 "(?<MONTH>[0-9]{2})"
-                                 "(?<YEAR>[0-9]{4})"
-                             ")",
-
-                             "(?:"                                   // either m-d-y
-                                 "(?<MONTH>[0-9]+)[-/.' ]+"
-                                 "(?<DAY>[0-9]+)[-/.' ]+"
-                                 "(?<YEAR>[0-9]+)"
-                             "|"                                     // or MMDDCCYY
-                                 "(?<MONTH>[0-9]{2})"
-                                 "(?<DAY>[0-9]{2})"
-                                 "(?<YEAR>[0-9]{4})"
-                             ")",
-
-                             "(?:"                                   // either d-m(-y)
-                                 "(?<DAY>[0-9]+)[-/.' ]+"
-                                 "(?<MONTH>[0-9]+)(?:[-/.' ]+"
-                                 "(?<YEAR>[0-9]+))?"
-                             "|"                                     // or DDMM(CCYY)
-                                 "(?<DAY>[0-9]{2})"
-                                 "(?<MONTH>[0-9]{2})"
-                                 "(?<YEAR>[0-9]+)?"
-                             ")",
-
-                             "(?:"                                   // either m-d(-y)
-                                 "(?<MONTH>[0-9]+)[-/.' ]+"
-                                 "(?<DAY>[0-9]+)(?:[-/.' ]+"
-                                 "(?<YEAR>[0-9]+))?"
-                             "|"                                     // or MMDD(CCYY)
-                                 "(?<MONTH>[0-9]{2})"
-                                 "(?<DAY>[0-9]{2})"
-                                 "(?<YEAR>[0-9]+)?"
-                             ")",
-};
-
-/** Parses a string into a date, given a format. This function
- * requires only knowing the order in which the year, month and day
- * appear. For example, 01-02-2003 will be parsed the same way as
- * 01/02/2003.
- * @param date_str The string containing a date being parsed
- * @param format An index specifying a format in date_format_user
- * @exception std::invalid_argument if the string can't be parsed into a date.
- * @return The parsed value of date_str on success, throws on failure
- */
-
-time64 parse_date (const std::string &date_str, int format)
-{
-    boost::regex r(date_regex[format]);
-    boost::smatch what;
-    if(!boost::regex_search(date_str, what, r))
-        throw std::invalid_argument (_("Value can't be parsed into a date using the selected date format."));  // regex didn't find a match
-
-    // Attention: different behavior from 2.6.x series !
-    // If date format without year was selected, the match
-    // should NOT have found a year.
-    if ((format >= 3) && (what.length("YEAR") != 0))
-        throw std::invalid_argument (_("Value appears to contain a year while the selected format forbids this."));
-
-    auto day = std::stoi (what.str("DAY"));
-    auto month = std::stoi (what.str("MONTH"));
-
-    int year;
-    if (format < 3)
-    {
-        /* The input dates have a year, so use that one */
-        year = std::stoi (what.str("YEAR"));
-
-        /* Handle two-digit years. */
-        if (year < 100)
-        {
-            /* We allow two-digit years in the range 1969 - 2068. */
-            if (year < 69)
-                year += 2000;
-            else
-                year += 1900;
-        }
-    }
-    else
-    {
-        /* The input dates don't have a year, so work with today's year.
-         */
-        gnc_timespec2dmy(timespec_now(), nullptr, nullptr, &year);
-    }
-
-    auto ts = gnc_dmy2timespec_neutral(day, month, year);
-    if (ts.tv_sec == INT64_MAX)
-        throw std::invalid_argument (_("Value can't be parsed into a date using the selected date format."));
-
-    return ts.tv_sec;
-}
-
 
 /** Convert str into a GncRational using the user-specified (import) currency format.
  * @param str The string to be parsed
@@ -317,7 +208,7 @@ void GncPreTrans::set (GncTransPropType prop_type, const std::string& value)
 
             case GncTransPropType::DATE:
                 m_date = boost::none;
-                m_date = parse_date (value, m_date_format); // Throws if parsing fails
+                m_date = GncDate(value, GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails
                 break;
 
             case GncTransPropType::NUM:
@@ -417,7 +308,8 @@ Transaction* GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency)
     auto trans = xaccMallocTransaction (book);
     xaccTransBeginEdit (trans);
     xaccTransSetCurrency (trans, m_commodity ? *m_commodity : currency);
-    xaccTransSetDatePostedSecsNormalized (trans, *m_date);
+    xaccTransSetDatePostedSecsNormalized (trans,
+                        static_cast<time64>(GncDateTime(*m_date, DayPart::neutral)));
 
     if (m_num)
         xaccTransSetNum (trans, m_num->c_str());
@@ -558,13 +450,15 @@ void GncPreSplit::set (GncTransPropType prop_type, const std::string& value)
             case GncTransPropType::REC_DATE:
                 m_rec_date = boost::none;
                 if (!value.empty())
-                    m_rec_date = parse_date (value, m_date_format); // Throws if parsing fails
+                    m_rec_date = GncDate (value,
+                                          GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails
                 break;
 
             case GncTransPropType::TREC_DATE:
                 m_trec_date = boost::none;
                 if (!value.empty())
-                    m_trec_date = parse_date (value, m_date_format); // Throws if parsing fails
+                    m_trec_date = GncDate (value,
+                                           GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails
                 break;
 
             default:
@@ -641,7 +535,7 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am
                             const boost::optional<std::string>& action,
                             const boost::optional<std::string>& memo,
                             const boost::optional<char>& rec_state,
-                            const boost::optional<time64> rec_date,
+                            const boost::optional<GncDate>& rec_date,
                             const boost::optional<GncNumeric> price)
 {
     QofBook* book = xaccTransGetBook (trans);
@@ -691,7 +585,8 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am
     if (rec_state && *rec_state != 'n')
         xaccSplitSetReconcile (split, *rec_state);
     if (rec_state && *rec_state == YREC && rec_date)
-        xaccSplitSetDateReconciledSecs (split, *rec_date);
+        xaccSplitSetDateReconciledSecs (split,
+                static_cast<time64>(GncDateTime(*rec_date, DayPart::neutral)));
 
 }
 

src/import-export/csv-imp/gnc-trans-props.hpp

@@ -41,6 +41,7 @@ extern "C" {
 #include <map>
 #include <memory>
 #include <boost/optional.hpp>
+#include <gnc-datetime.hpp>
 #include <gnc-numeric.hpp>
 
 /** Enumeration for column types. These are the different types of
@@ -103,7 +104,6 @@ private:
 GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split);
 
 
-time64 parse_date (const std::string &date_str, int format);
 gnc_commodity* parse_commodity (const std::string& comm_str);
 GncNumeric parse_amount (const std::string &str, int currency_format);
 
@@ -139,7 +139,7 @@ public:
 private:
     int m_date_format;
     boost::optional<std::string> m_differ;
-    boost::optional<time64> m_date;
+    boost::optional<GncDate> m_date;
     boost::optional<std::string> m_num;
     boost::optional<std::string> m_desc;
     boost::optional<std::string> m_notes;
@@ -176,12 +176,12 @@ private:
     boost::optional<GncNumeric> m_price;
     boost::optional<std::string> m_memo;
     boost::optional<char> m_rec_state;
-    boost::optional<time64> m_rec_date;
+    boost::optional<GncDate> m_rec_date;
     boost::optional<std::string> m_taction;
     boost::optional<Account*> m_taccount;
     boost::optional<std::string> m_tmemo;
     boost::optional<char> m_trec_state;
-    boost::optional<time64> m_trec_date;
+    boost::optional<GncDate> m_trec_date;
     bool created = false;
 
     std::map<GncTransPropType, std::string> m_errors;

src/import-export/csv-imp/gnc-tx-import.cpp

@@ -43,14 +43,6 @@ extern "C" {
 
 G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT;
 
-const int num_date_formats = 5;
-const gchar* date_format_user[] = {N_("y-m-d"),
-                                   N_("d-m-y"),
-                                   N_("m-d-y"),
-                                   N_("d-m"),
-                                   N_("m-d")
-                                  };
-
 const int num_currency_formats = 3;
 const gchar* currency_format_user[] = {N_("Locale"),
                                        N_("Period: 123,456.78"),
@@ -742,6 +734,8 @@ void GncTxImport::update_pre_trans_props (uint32_t row, uint32_t col, GncTransPr
     if ((prop_type == GncTransPropType::NONE) || (prop_type > GncTransPropType::TRANS_PROPS))
         return; /* Only deal with transaction related properties. */
 
+    /* Deliberately make a copy of the GncPreTrans. It may be the original one was shared
+     * with a previous line and should no longer be after the transprop is changed. */
     auto trans_props = std::make_shared<GncPreTrans> (*(std::get<PL_PRETRANS>(m_parsed_lines[row])).get());
     auto value = std::string();
 

src/import-export/csv-imp/gnc-tx-import.hpp

@@ -64,10 +64,6 @@ struct DraftTransaction
 extern const int num_currency_formats;
 extern const gchar* currency_format_user[];
 
-/* A set of date formats that the user sees. */
-extern const int num_date_formats;
-extern const gchar* date_format_user[];
-
 /** An enum describing the columns found in a parse_line_t. Currently these are:
  *  - a tokenized line of input
  *  - an optional error string

src/import-export/csv-imp/test/test-tx-import.cpp

@@ -62,150 +62,3 @@ public:
 protected:
     std::unique_ptr<GncTxImport> tx_importer;
 };
-
-
-/* parse_date
-time64 parse_date (const char* date_str, int format)// C: 14 in 7 SCM: 9 in 2 Local: 1:0:0
-*/
-TEST(GncTxImportTest, parse_date)
-{
-    time64 rawtime = gnc_time (nullptr);
-    struct tm *tm = gnc_gmtime (&rawtime);
-    int curr_year = tm->tm_year;
-
-
-    /* Note: tm_year = year - 1900 and tm_mon = 0-11
-     * I'm writing the expected values as subtractions for easier
-     * comparison with the date string under test
-     */
-    parse_date_data test_dates[] =
-    {
-        // supported combinations  -/.'
-        { 0, "2013-08-01", 2013 - 1900,  8 - 1,  1},
-        { 0,  "2013-8-01", 2013 - 1900,  8 - 1,  1},
-        { 0,  "2013-08-1", 2013 - 1900,  8 - 1,  1},
-        { 0,   "2013-8-1", 2013 - 1900,  8 - 1,  1},
-        { 0,   "13-08-01", 2013 - 1900,  8 - 1,  1},
-        { 0,    "13-8-01", 2013 - 1900,  8 - 1,  1},
-        { 0,    "13-08-1", 2013 - 1900,  8 - 1,  1},
-        { 0,     "13-8-1", 2013 - 1900,  8 - 1,  1},
-        { 0, "2009/11/04", 2009 - 1900, 11 - 1,  4},
-        { 0,  "1985.3.12", 1985 - 1900,  3 - 1, 12},
-        { 0,      "3'6'8", 2003 - 1900,  6 - 1,  8},
-        { 0,   "20130801", 2013 - 1900,  8 - 1,  1},
-        { 1, "01-08-2013", 2013 - 1900,  8 - 1,  1},
-        { 1,  "01-8-2013", 2013 - 1900,  8 - 1,  1},
-        { 1,  "1-08-2013", 2013 - 1900,  8 - 1,  1},
-        { 1,   "1-8-2013", 2013 - 1900,  8 - 1,  1},
-        { 1,   "01-08-13", 2013 - 1900,  8 - 1,  1},
-        { 1,    "01-8-13", 2013 - 1900,  8 - 1,  1},
-        { 1,    "1-08-13", 2013 - 1900,  8 - 1,  1},
-        { 1,     "1-8-13", 2013 - 1900,  8 - 1,  1},
-        { 1, "04/11/2009", 2009 - 1900, 11 - 1,  4},
-        { 1,  "12.3.1985", 1985 - 1900,  3 - 1, 12},
-        { 1,      "8'6'3", 2003 - 1900,  6 - 1,  8},
-        { 1,   "01082013", 2013 - 1900,  8 - 1,  1},
-        { 2, "08-01-2013", 2013 - 1900,  8 - 1,  1},
-        { 2,  "8-01-2013", 2013 - 1900,  8 - 1,  1},
-        { 2,  "08-1-2013", 2013 - 1900,  8 - 1,  1},
-        { 2,   "8-1-2013", 2013 - 1900,  8 - 1,  1},
-        { 2,   "08-01-13", 2013 - 1900,  8 - 1,  1},
-        { 2,    "8-01-13", 2013 - 1900,  8 - 1,  1},
-        { 2,    "08-1-13", 2013 - 1900,  8 - 1,  1},
-        { 2,     "8-1-13", 2013 - 1900,  8 - 1,  1},
-        { 2, "11/04/2009", 2009 - 1900, 11 - 1,  4},
-        { 2,  "3.12.1985", 1985 - 1900,  3 - 1, 12},
-        { 2,      "6'8'3", 2003 - 1900,  6 - 1,  8},
-        { 2,   "08012013", 2013 - 1900,  8 - 1,  1},
-        { 3,      "01-08",   curr_year,  8 - 1,  1},
-        { 3,       "01-8",   curr_year,  8 - 1,  1},
-        { 3,       "1-08",   curr_year,  8 - 1,  1},
-        { 3,        "1-8",   curr_year,  8 - 1,  1},
-        { 3,      "04/11",   curr_year, 11 - 1,  4},
-        { 3,       "12.3",   curr_year,  3 - 1, 12},
-        { 3,        "8'6",   curr_year,  6 - 1,  8},
-        { 3,       "0108",   curr_year,  8 - 1,  1},
-        { 4,      "08-01",   curr_year,  8 - 1,  1},
-        { 4,       "8-01",   curr_year,  8 - 1,  1},
-        { 4,       "08-1",   curr_year,  8 - 1,  1},
-        { 4,        "8-1",   curr_year,  8 - 1,  1},
-        { 4,      "11/04",   curr_year, 11 - 1,  4},
-        { 4,       "3.12",   curr_year,  3 - 1, 12},
-        { 4,        "6'8",   curr_year,  6 - 1,  8},
-        { 4,       "0801",   curr_year,  8 - 1,  1},
-
-        // ambiguous date formats
-        // current parser doesn't know how to disambiguate
-        // and hence refuses to parse
-        // can possibly improved with a smarter parser
-        { 0,     "130801",          -1,     -1, -1},
-        { 1,     "010813",          -1,     -1, -1},
-        { 2,     "080113",          -1,     -1, -1},
-
-        // Combinations that don't make sense
-        // but can still be entered by a user
-        // Should ideally all result in refusal to parse...
-        { 0,      "08-01",          -1,     -1, -1},
-        { 0,       "0801",          -1,     -1, -1},
-        { 1,      "01-08",          -1,     -1, -1},
-        { 1,       "0108",          -1,     -1, -1},
-        { 2,      "08-01",          -1,     -1, -1},
-        { 2,       "0801",          -1,     -1, -1},
-        { 3, "01-08-2013",          -1,     -1, -1},
-        { 3,   "01-08-13",          -1,     -1, -1},
-        { 3,   "08-08-08",          -1,     -1, -1},
-        { 3,   "01082013",          -1,     -1, -1},
-        { 3,     "010813",          -1,     -1, -1},
-        { 3,   "20130108",          -1,     -1, -1},
-        { 4, "08-01-2013",          -1,     -1, -1},
-        { 4,   "08-01-13",          -1,     -1, -1},
-        { 4, "2013-08-01",          -1,     -1, -1},
-        { 4,   "09-08-01",          -1,     -1, -1},
-        { 4,   "08012013",          -1,     -1, -1},
-        { 4,     "080113",          -1,     -1, -1},
-        { 4,   "20130801",          -1,     -1, -1},
-
-        // Sentinel to mark the end of available tests
-        { 0,         NULL,           0,      0,  0},
-
-    };
-    int i = 0;
-
-    gnc_tm_free(tm);
-    while (test_dates[i].date_str)
-    {
-        gboolean success = TRUE;
-        int got_year = 0, got_month = 0, got_day = 0;
-
-        try
-        {
-            rawtime = parse_date (std::string(test_dates[i].date_str), test_dates[i].date_fmt);
-            tm = gnc_gmtime (&rawtime);
-            got_year = tm->tm_year;
-            got_month = tm->tm_mon;
-            got_day = tm->tm_mday;
-            gnc_tm_free(tm);
-        }
-        catch (std::invalid_argument)
-        {
-            got_year = got_month = got_day = -1;
-        }
-
-        if ((got_year  != test_dates[i].exp_year) ||
-            (got_month != test_dates[i].exp_month) ||
-            (got_day   != test_dates[i].exp_day))
-        {
-            g_error ("Parse_date failed for date '%s' and format '%d'.\n"
-                            "Expected result: year %d, month %d, day %d\n"
-                            "Obtained result: year %d, month %d, day %d",
-                            test_dates[i].date_str,
-                            test_dates[i].date_fmt,
-                            test_dates[i].exp_year,
-                            test_dates[i].exp_month,
-                            test_dates[i].exp_day,
-                            got_year, got_month, got_day);
-        }
-
-        i++;
-    }
-}