IntenseFinance/gnucash
3
0
Fork 0
mirror of https://github.com/Gnucash/gnucash.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Use GncDate in csv importer

Browse Source 
This removes all date specific functionality from the importer
This commit is contained in:
Geert Janssens 2017-05-01 20:19:49 +02:00
parent 7df29b572a
commit 9af57849ba
6 changed files with 19 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/import-export/csv-imp/assistant-csv-trans-import.cpp
View File

@ -521,10 +521,8 @@ CsvImpTransAssist::CsvImpTransAssist ()
/* Add in the date format combo box and hook it up to an event handler. */
date_format_combo = GTK_COMBO_BOX_TEXT(gtk_combo_box_text_new());
for (int i = 0; i < num_date_formats; i++)
{
gtk_combo_box_text_append_text (date_format_combo, _(date_format_user[i]));
}
for (auto& date_fmt : GncDate::c_formats)
gtk_combo_box_text_append_text (date_format_combo, _(date_fmt.m_fmt.c_str()));
gtk_combo_box_set_active (GTK_COMBO_BOX(date_format_combo), 0);
g_signal_connect (G_OBJECT(date_format_combo), "changed",
G_CALLBACK(csv_tximp_preview_date_fmt_sel_cb), this);
@ -1560,7 +1558,7 @@ CsvImpTransAssist::preview_refresh ()
gtk_toggle_button_set_active (GTK_TOGGLE_BUTTON(fixed_button),
(tx_imp->file_format() != GncImpFileFormat::CSV));
// This section deals with the combo's and character encoding
// Set Date & Currency Format and Character encoding
gtk_combo_box_set_active (GTK_COMBO_BOX(date_format_combo),
tx_imp->date_format());
gtk_combo_box_set_active (GTK_COMBO_BOX(currency_format_combo),

125
src/import-export/csv-imp/gnc-trans-props.cpp
View File

@ -93,115 +93,6 @@ GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split)
return GncTransPropType::NONE;
}
/* Regular expressions used to parse dates per date format */
const char* date_regex[] = {
"(?:" // either y-m-d
"(?<YEAR>[0-9]+)[-/.' ]+"
"(?<MONTH>[0-9]+)[-/.' ]+"
"(?<DAY>[0-9]+)"
"|" // or CCYYMMDD
"(?<YEAR>[0-9]{4})"
"(?<MONTH>[0-9]{2})"
"(?<DAY>[0-9]{2})"
")",
"(?:" // either d-m-y
"(?<DAY>[0-9]+)[-/.' ]+"
"(?<MONTH>[0-9]+)[-/.' ]+"
"(?<YEAR>[0-9]+)"
"|" // or DDMMCCYY
"(?<DAY>[0-9]{2})"
"(?<MONTH>[0-9]{2})"
"(?<YEAR>[0-9]{4})"
")",
"(?:" // either m-d-y
"(?<MONTH>[0-9]+)[-/.' ]+"
"(?<DAY>[0-9]+)[-/.' ]+"
"(?<YEAR>[0-9]+)"
"|" // or MMDDCCYY
"(?<MONTH>[0-9]{2})"
"(?<DAY>[0-9]{2})"
"(?<YEAR>[0-9]{4})"
")",
"(?:" // either d-m(-y)
"(?<DAY>[0-9]+)[-/.' ]+"
"(?<MONTH>[0-9]+)(?:[-/.' ]+"
"(?<YEAR>[0-9]+))?"
"|" // or DDMM(CCYY)
"(?<DAY>[0-9]{2})"
"(?<MONTH>[0-9]{2})"
"(?<YEAR>[0-9]+)?"
")",
"(?:" // either m-d(-y)
"(?<MONTH>[0-9]+)[-/.' ]+"
"(?<DAY>[0-9]+)(?:[-/.' ]+"
"(?<YEAR>[0-9]+))?"
"|" // or MMDD(CCYY)
"(?<MONTH>[0-9]{2})"
"(?<DAY>[0-9]{2})"
"(?<YEAR>[0-9]+)?"
")",
};
/** Parses a string into a date, given a format. This function
* requires only knowing the order in which the year, month and day
* appear. For example, 01-02-2003 will be parsed the same way as
* 01/02/2003.
* @param date_str The string containing a date being parsed
* @param format An index specifying a format in date_format_user
* @exception std::invalid_argument if the string can't be parsed into a date.
* @return The parsed value of date_str on success, throws on failure
*/
time64 parse_date (const std::string &date_str, int format)
{
boost::regex r(date_regex[format]);
boost::smatch what;
if(!boost::regex_search(date_str, what, r))
throw std::invalid_argument (_("Value can't be parsed into a date using the selected date format.")); // regex didn't find a match
// Attention: different behavior from 2.6.x series !
// If date format without year was selected, the match
// should NOT have found a year.
if ((format >= 3) && (what.length("YEAR") != 0))
throw std::invalid_argument (_("Value appears to contain a year while the selected format forbids this."));
auto day = std::stoi (what.str("DAY"));
auto month = std::stoi (what.str("MONTH"));
int year;
if (format < 3)
{
/* The input dates have a year, so use that one */
year = std::stoi (what.str("YEAR"));
/* Handle two-digit years. */
if (year < 100)
{
/* We allow two-digit years in the range 1969 - 2068. */
if (year < 69)
year += 2000;
else
year += 1900;
}
}
else
{
/* The input dates don't have a year, so work with today's year.
*/
gnc_timespec2dmy(timespec_now(), nullptr, nullptr, &year);
}
auto ts = gnc_dmy2timespec_neutral(day, month, year);
if (ts.tv_sec == INT64_MAX)
throw std::invalid_argument (_("Value can't be parsed into a date using the selected date format."));
return ts.tv_sec;
}
/** Convert str into a GncRational using the user-specified (import) currency format.
* @param str The string to be parsed
@ -317,7 +208,7 @@ void GncPreTrans::set (GncTransPropType prop_type, const std::string& value)
case GncTransPropType::DATE:
m_date = boost::none;
m_date = parse_date (value, m_date_format); // Throws if parsing fails
m_date = GncDate(value, GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails
break;
case GncTransPropType::NUM:
@ -417,7 +308,8 @@ Transaction* GncPreTrans::create_trans (QofBook* book, gnc_commodity* currency)
auto trans = xaccMallocTransaction (book);
xaccTransBeginEdit (trans);
xaccTransSetCurrency (trans, m_commodity ? *m_commodity : currency);
xaccTransSetDatePostedSecsNormalized (trans, *m_date);
xaccTransSetDatePostedSecsNormalized (trans,
static_cast<time64>(GncDateTime(*m_date, DayPart::neutral)));
if (m_num)
xaccTransSetNum (trans, m_num->c_str());
@ -558,13 +450,15 @@ void GncPreSplit::set (GncTransPropType prop_type, const std::string& value)
case GncTransPropType::REC_DATE:
m_rec_date = boost::none;
if (!value.empty())
m_rec_date = parse_date (value, m_date_format); // Throws if parsing fails
m_rec_date = GncDate (value,
GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails
break;
case GncTransPropType::TREC_DATE:
m_trec_date = boost::none;
if (!value.empty())
m_trec_date = parse_date (value, m_date_format); // Throws if parsing fails
m_trec_date = GncDate (value,
GncDate::c_formats[m_date_format].m_fmt); // Throws if parsing fails
break;
default:
@ -641,7 +535,7 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am
const boost::optional<std::string>& action,
const boost::optional<std::string>& memo,
const boost::optional<char>& rec_state,
const boost::optional<time64> rec_date,
const boost::optional<GncDate>& rec_date,
const boost::optional<GncNumeric> price)
{
QofBook* book = xaccTransGetBook (trans);
@ -691,7 +585,8 @@ static void trans_add_split (Transaction* trans, Account* account, GncNumeric am
if (rec_state && *rec_state != 'n')
xaccSplitSetReconcile (split, *rec_state);
if (rec_state && *rec_state == YREC && rec_date)
xaccSplitSetDateReconciledSecs (split, *rec_date);
xaccSplitSetDateReconciledSecs (split,
static_cast<time64>(GncDateTime(*rec_date, DayPart::neutral)));
}

8
src/import-export/csv-imp/gnc-trans-props.hpp
View File

@ -41,6 +41,7 @@ extern "C" {
#include <map>
#include <memory>
#include <boost/optional.hpp>
#include <gnc-datetime.hpp>
#include <gnc-numeric.hpp>
/** Enumeration for column types. These are the different types of
@ -103,7 +104,6 @@ private:
GncTransPropType sanitize_trans_prop (GncTransPropType prop, bool multi_split);
time64 parse_date (const std::string &date_str, int format);
gnc_commodity* parse_commodity (const std::string& comm_str);
GncNumeric parse_amount (const std::string &str, int currency_format);
@ -139,7 +139,7 @@ public:
private:
int m_date_format;
boost::optional<std::string> m_differ;
boost::optional<time64> m_date;
boost::optional<GncDate> m_date;
boost::optional<std::string> m_num;
boost::optional<std::string> m_desc;
boost::optional<std::string> m_notes;
@ -176,12 +176,12 @@ private:
boost::optional<GncNumeric> m_price;
boost::optional<std::string> m_memo;
boost::optional<char> m_rec_state;
boost::optional<time64> m_rec_date;
boost::optional<GncDate> m_rec_date;
boost::optional<std::string> m_taction;
boost::optional<Account*> m_taccount;
boost::optional<std::string> m_tmemo;
boost::optional<char> m_trec_state;
boost::optional<time64> m_trec_date;
boost::optional<GncDate> m_trec_date;
bool created = false;
std::map<GncTransPropType, std::string> m_errors;

10
src/import-export/csv-imp/gnc-tx-import.cpp
View File

@ -43,14 +43,6 @@ extern "C" {
G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT;
const int num_date_formats = 5;
const gchar* date_format_user[] = {N_("y-m-d"),
N_("d-m-y"),
N_("m-d-y"),
N_("d-m"),
N_("m-d")
};
const int num_currency_formats = 3;
const gchar* currency_format_user[] = {N_("Locale"),
N_("Period: 123,456.78"),
@ -742,6 +734,8 @@ void GncTxImport::update_pre_trans_props (uint32_t row, uint32_t col, GncTransPr
if ((prop_type == GncTransPropType::NONE) || (prop_type > GncTransPropType::TRANS_PROPS))
return; /* Only deal with transaction related properties. */
/* Deliberately make a copy of the GncPreTrans. It may be the original one was shared
* with a previous line and should no longer be after the transprop is changed. */
auto trans_props = std::make_shared<GncPreTrans> (*(std::get<PL_PRETRANS>(m_parsed_lines[row])).get());
auto value = std::string();

4
src/import-export/csv-imp/gnc-tx-import.hpp
View File

@ -64,10 +64,6 @@ struct DraftTransaction
extern const int num_currency_formats;
extern const gchar* currency_format_user[];
/* A set of date formats that the user sees. */
extern const int num_date_formats;
extern const gchar* date_format_user[];
/** An enum describing the columns found in a parse_line_t. Currently these are:
* - a tokenized line of input
* - an optional error string

147
src/import-export/csv-imp/test/test-tx-import.cpp
View File

@ -62,150 +62,3 @@ public:
protected:
std::unique_ptr<GncTxImport> tx_importer;
};
/* parse_date
time64 parse_date (const char* date_str, int format)// C: 14 in 7 SCM: 9 in 2 Local: 1:0:0
*/
TEST(GncTxImportTest, parse_date)
{
time64 rawtime = gnc_time (nullptr);
struct tm *tm = gnc_gmtime (&rawtime);
int curr_year = tm->tm_year;
/* Note: tm_year = year - 1900 and tm_mon = 0-11
* I'm writing the expected values as subtractions for easier
* comparison with the date string under test
*/
parse_date_data test_dates[] =
{
// supported combinations -/.'
{ 0, "2013-08-01", 2013 - 1900, 8 - 1, 1},
{ 0, "2013-8-01", 2013 - 1900, 8 - 1, 1},
{ 0, "2013-08-1", 2013 - 1900, 8 - 1, 1},
{ 0, "2013-8-1", 2013 - 1900, 8 - 1, 1},
{ 0, "13-08-01", 2013 - 1900, 8 - 1, 1},
{ 0, "13-8-01", 2013 - 1900, 8 - 1, 1},
{ 0, "13-08-1", 2013 - 1900, 8 - 1, 1},
{ 0, "13-8-1", 2013 - 1900, 8 - 1, 1},
{ 0, "2009/11/04", 2009 - 1900, 11 - 1, 4},
{ 0, "1985.3.12", 1985 - 1900, 3 - 1, 12},
{ 0, "3'6'8", 2003 - 1900, 6 - 1, 8},
{ 0, "20130801", 2013 - 1900, 8 - 1, 1},
{ 1, "01-08-2013", 2013 - 1900, 8 - 1, 1},
{ 1, "01-8-2013", 2013 - 1900, 8 - 1, 1},
{ 1, "1-08-2013", 2013 - 1900, 8 - 1, 1},
{ 1, "1-8-2013", 2013 - 1900, 8 - 1, 1},
{ 1, "01-08-13", 2013 - 1900, 8 - 1, 1},
{ 1, "01-8-13", 2013 - 1900, 8 - 1, 1},
{ 1, "1-08-13", 2013 - 1900, 8 - 1, 1},
{ 1, "1-8-13", 2013 - 1900, 8 - 1, 1},
{ 1, "04/11/2009", 2009 - 1900, 11 - 1, 4},
{ 1, "12.3.1985", 1985 - 1900, 3 - 1, 12},
{ 1, "8'6'3", 2003 - 1900, 6 - 1, 8},
{ 1, "01082013", 2013 - 1900, 8 - 1, 1},
{ 2, "08-01-2013", 2013 - 1900, 8 - 1, 1},
{ 2, "8-01-2013", 2013 - 1900, 8 - 1, 1},
{ 2, "08-1-2013", 2013 - 1900, 8 - 1, 1},
{ 2, "8-1-2013", 2013 - 1900, 8 - 1, 1},
{ 2, "08-01-13", 2013 - 1900, 8 - 1, 1},
{ 2, "8-01-13", 2013 - 1900, 8 - 1, 1},
{ 2, "08-1-13", 2013 - 1900, 8 - 1, 1},
{ 2, "8-1-13", 2013 - 1900, 8 - 1, 1},
{ 2, "11/04/2009", 2009 - 1900, 11 - 1, 4},
{ 2, "3.12.1985", 1985 - 1900, 3 - 1, 12},
{ 2, "6'8'3", 2003 - 1900, 6 - 1, 8},
{ 2, "08012013", 2013 - 1900, 8 - 1, 1},
{ 3, "01-08", curr_year, 8 - 1, 1},
{ 3, "01-8", curr_year, 8 - 1, 1},
{ 3, "1-08", curr_year, 8 - 1, 1},
{ 3, "1-8", curr_year, 8 - 1, 1},
{ 3, "04/11", curr_year, 11 - 1, 4},
{ 3, "12.3", curr_year, 3 - 1, 12},
{ 3, "8'6", curr_year, 6 - 1, 8},
{ 3, "0108", curr_year, 8 - 1, 1},
{ 4, "08-01", curr_year, 8 - 1, 1},
{ 4, "8-01", curr_year, 8 - 1, 1},
{ 4, "08-1", curr_year, 8 - 1, 1},
{ 4, "8-1", curr_year, 8 - 1, 1},
{ 4, "11/04", curr_year, 11 - 1, 4},
{ 4, "3.12", curr_year, 3 - 1, 12},
{ 4, "6'8", curr_year, 6 - 1, 8},
{ 4, "0801", curr_year, 8 - 1, 1},
// ambiguous date formats
// current parser doesn't know how to disambiguate
// and hence refuses to parse
// can possibly improved with a smarter parser
{ 0, "130801", -1, -1, -1},
{ 1, "010813", -1, -1, -1},
{ 2, "080113", -1, -1, -1},
// Combinations that don't make sense
// but can still be entered by a user
// Should ideally all result in refusal to parse...
{ 0, "08-01", -1, -1, -1},
{ 0, "0801", -1, -1, -1},
{ 1, "01-08", -1, -1, -1},
{ 1, "0108", -1, -1, -1},
{ 2, "08-01", -1, -1, -1},
{ 2, "0801", -1, -1, -1},
{ 3, "01-08-2013", -1, -1, -1},
{ 3, "01-08-13", -1, -1, -1},
{ 3, "08-08-08", -1, -1, -1},
{ 3, "01082013", -1, -1, -1},
{ 3, "010813", -1, -1, -1},
{ 3, "20130108", -1, -1, -1},
{ 4, "08-01-2013", -1, -1, -1},
{ 4, "08-01-13", -1, -1, -1},
{ 4, "2013-08-01", -1, -1, -1},
{ 4, "09-08-01", -1, -1, -1},
{ 4, "08012013", -1, -1, -1},
{ 4, "080113", -1, -1, -1},
{ 4, "20130801", -1, -1, -1},
// Sentinel to mark the end of available tests
{ 0, NULL, 0, 0, 0},
};
int i = 0;
gnc_tm_free(tm);
while (test_dates[i].date_str)
{
gboolean success = TRUE;
int got_year = 0, got_month = 0, got_day = 0;
try
{
rawtime = parse_date (std::string(test_dates[i].date_str), test_dates[i].date_fmt);
tm = gnc_gmtime (&rawtime);
got_year = tm->tm_year;
got_month = tm->tm_mon;
got_day = tm->tm_mday;
gnc_tm_free(tm);
}
catch (std::invalid_argument)
{
got_year = got_month = got_day = -1;
}
if ((got_year != test_dates[i].exp_year) ||
(got_month != test_dates[i].exp_month) ||
(got_day != test_dates[i].exp_day))
{
g_error ("Parse_date failed for date '%s' and format '%d'.\n"
"Expected result: year %d, month %d, day %d\n"
"Obtained result: year %d, month %d, day %d",
test_dates[i].date_str,
test_dates[i].date_fmt,
test_dates[i].exp_year,
test_dates[i].exp_month,
test_dates[i].exp_day,
got_year, got_month, got_day);
}
i++;
}
}