diff --git a/gnucash/import-export/csv-imp/CMakeLists.txt b/gnucash/import-export/csv-imp/CMakeLists.txt index e2fa97fee9..0bc7d2938f 100644 --- a/gnucash/import-export/csv-imp/CMakeLists.txt +++ b/gnucash/import-export/csv-imp/CMakeLists.txt @@ -19,6 +19,7 @@ SET(csv_import_SOURCES gnc-csv-trans-settings.cpp gnc-dummy-tokenizer.cpp gnc-fw-tokenizer.cpp + gnc-price-import.cpp gnc-price-props.cpp gnc-tokenizer.cpp gnc-trans-props.cpp @@ -45,6 +46,7 @@ SET(csv_import_noinst_HEADERS gnc-csv-trans-settings.hpp gnc-dummy-tokenizer.hpp gnc-fw-tokenizer.hpp + gnc-price-import.hpp gnc-price-props.hpp gnc-tokenizer.hpp gnc-trans-props.hpp diff --git a/gnucash/import-export/csv-imp/Makefile.am b/gnucash/import-export/csv-imp/Makefile.am index 8b13d01e18..408396331a 100644 --- a/gnucash/import-export/csv-imp/Makefile.am +++ b/gnucash/import-export/csv-imp/Makefile.am @@ -13,6 +13,7 @@ libgncmod_csv_import_la_SOURCES = \ gnc-csv-gnumeric-popup.c \ gnc-dummy-tokenizer.cpp \ gnc-fw-tokenizer.cpp \ + gnc-price-import.cpp \ gnc-price-props.cpp \ gnc-tokenizer.cpp \ gnc-tx-import.cpp \ @@ -29,6 +30,7 @@ noinst_HEADERS = \ gnc-csv-gnumeric-popup.h \ gnc-dummy-tokenizer.hpp \ gnc-fw-tokenizer.hpp \ + gnc-price-import.hpp \ gnc-price-props.hpp \ gnc-tokenizer.hpp \ gnc-tx-import.hpp \ diff --git a/gnucash/import-export/csv-imp/gnc-csv-trans-settings.cpp b/gnucash/import-export/csv-imp/gnc-csv-trans-settings.cpp index 6611875380..48cfdce9d2 100644 --- a/gnucash/import-export/csv-imp/gnc-csv-trans-settings.cpp +++ b/gnucash/import-export/csv-imp/gnc-csv-trans-settings.cpp @@ -108,6 +108,14 @@ static std::shared_ptr create_int_gnc_exp_preset(void) GncTransPropType::PRICE }; + preset->m_column_types_price = { + GncPricePropType::DATE, + GncPricePropType::AMOUNT, + GncPricePropType::CURRENCY_FROM, + GncPricePropType::CURRENCY_TO, + GncPricePropType::SYMBOL_FROM + }; + return preset; } diff --git a/gnucash/import-export/csv-imp/gnc-csv-trans-settings.hpp b/gnucash/import-export/csv-imp/gnc-csv-trans-settings.hpp index 379c6600d8..93275f2da0 100644 --- a/gnucash/import-export/csv-imp/gnc-csv-trans-settings.hpp +++ b/gnucash/import-export/csv-imp/gnc-csv-trans-settings.hpp @@ -36,6 +36,7 @@ extern "C" { #include #include #include "gnc-trans-props.hpp" +#include "gnc-price-props.hpp" #include "gnc-tokenizer.hpp" /** Enumeration for separator checkbutton types. These are the @@ -93,7 +94,8 @@ std::string m_separators; // Separators for csv format Account *m_base_account; // Base account std::vector m_column_types; // The Column types in order -std::vector m_column_widths; // The Column widths +std::vector m_column_types_price; // The Column Price types in order +std::vector m_column_widths; // The Column widths bool m_load_error; // Was there an error while parsing the state file ? }; diff --git a/gnucash/import-export/csv-imp/gnc-price-import.cpp b/gnucash/import-export/csv-imp/gnc-price-import.cpp new file mode 100644 index 0000000000..09deb934e0 --- /dev/null +++ b/gnucash/import-export/csv-imp/gnc-price-import.cpp @@ -0,0 +1,654 @@ +/********************************************************************\ + * gnc-price-import.cpp - import prices from csv files * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License* + * along with this program; if not, contact: * + * * + * Free Software Foundation Voice: +1-617-542-5942 * + * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 * + * Boston, MA 02110-1301, USA gnu@gnu.org * + * * +\********************************************************************/ + +#include + +extern "C" { +#include +#if PLATFORM(WINDOWS) +#include +#endif + +#include + +#include "gnc-ui-util.h" //get book +#include "gnc-commodity.h" +#include "gnc-pricedb.h" +} + +#include +#include + +#include "gnc-price-import.hpp" +#include "gnc-price-props.hpp" +#include "gnc-csv-tokenizer.hpp" +#include "gnc-fw-tokenizer.hpp" +#include "gnc-csv-trans-settings.hpp" + +G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT; + +const int num_date_formats_price = 5; +const gchar* date_format_user_price[] = {N_("y-m-d"), + N_("d-m-y"), + N_("m-d-y"), + N_("d-m"), + N_("m-d") + }; + +const int num_currency_formats_price = 3; +const gchar* currency_format_user_price[] = {N_("Locale"), + N_("Period: 123,456.78"), + N_("Comma: 123.456,78") + }; + + +/** Constructor for GncPriceImport. + * @return Pointer to a new GncCSvParseData + */ +GncPriceImport::GncPriceImport(GncImpFileFormat format) +{ + /* All of the data pointers are initially NULL. This is so that, if + * gnc_csv_parse_data_free is called before all of the data is + * initialized, only the data that needs to be freed is freed. */ + m_skip_errors = false; + file_format(m_settings.m_file_format = format); +} + +/** Destructor for GncPriceImport. + */ +GncPriceImport::~GncPriceImport() +{ +} + +/** Sets the file format for the file to import, which + * may cause the file to be reloaded as well if the + * previously set file format was different and a + * filename was already set. + * @param format the new format to set + * @exception std::ifstream::failure if file reloading fails + */ +void GncPriceImport::file_format(GncImpFileFormat format) +{ + if (m_tokenizer && m_settings.m_file_format == format) + return; + + auto new_encoding = std::string("UTF-8"); + auto new_imp_file = std::string(); + + // Recover common settings from old tokenizer + if (m_tokenizer) + { + new_encoding = m_tokenizer->encoding(); + new_imp_file = m_tokenizer->current_file(); + if (file_format() == GncImpFileFormat::FIXED_WIDTH) + { + auto fwtok = dynamic_cast(m_tokenizer.get()); + if (!fwtok->get_columns().empty()) + m_settings.m_column_widths = fwtok->get_columns(); + } + } + + m_settings.m_file_format = format; + m_tokenizer = gnc_tokenizer_factory(m_settings.m_file_format); + + // Set up new tokenizer with common settings + // recovered from old tokenizer + m_tokenizer->encoding(new_encoding); + load_file(new_imp_file); + + // Restore potentially previously set separators or column_widths + if ((file_format() == GncImpFileFormat::CSV) + && !m_settings.m_separators.empty()) + separators (m_settings.m_separators); + else if ((file_format() == GncImpFileFormat::FIXED_WIDTH) + && !m_settings.m_column_widths.empty()) + { + auto fwtok = dynamic_cast(m_tokenizer.get()); + fwtok->columns (m_settings.m_column_widths); + } + +} + +GncImpFileFormat GncPriceImport::file_format() +{ + return m_settings.m_file_format; +} + +void GncPriceImport::over_write (bool over) +{ + m_over_write = over; +} + +bool GncPriceImport::over_write () { return m_over_write; } + +void GncPriceImport::reset_formatted_column (std::vector& col_types) +{ + for (auto col_type: col_types) + { + auto col = std::find (m_settings.m_column_types_price.begin(), + m_settings.m_column_types_price.end(), col_type); + if (col != m_settings.m_column_types_price.end()) + set_column_type_price (col - m_settings.m_column_types_price.begin(), col_type, true); + } +} + +void GncPriceImport::currency_format (int currency_format) +{ + m_settings.m_currency_format = currency_format; + + /* Reparse all currency related columns */ + std::vector commodities = { GncPricePropType::AMOUNT }; + reset_formatted_column (commodities); +} +int GncPriceImport::currency_format () { return m_settings.m_currency_format; } + +void GncPriceImport::date_format (int date_format) +{ + m_settings.m_date_format = date_format; + + /* Reparse all date related columns */ + std::vector dates = { GncPricePropType::DATE }; + reset_formatted_column (dates); +} +int GncPriceImport::date_format () { return m_settings.m_date_format; } + +/** Converts raw file data using a new encoding. This function must be + * called after load_file only if load_file guessed + * the wrong encoding. + * @param encoding Encoding that data should be translated using + */ +void GncPriceImport::encoding (const std::string& encoding) +{ + + // TODO investigate if we can catch conversion errors and report them + if (m_tokenizer) + { + m_tokenizer->encoding(encoding); // May throw + try + { + tokenize(false); + } + catch (...) + { }; + } + + m_settings.m_encoding = encoding; +} + +std::string GncPriceImport::encoding () { return m_settings.m_encoding; } + +void GncPriceImport::update_skipped_lines(boost::optional start, boost::optional end, + boost::optional alt, boost::optional errors) +{ + if (start) + m_settings.m_skip_start_lines = *start; + if (end) + m_settings.m_skip_end_lines = *end; + if (alt) + m_settings.m_skip_alt_lines = *alt; + if (errors) + m_skip_errors = *errors; + + for (uint32_t i = 0; i < m_parsed_lines.size(); i++) + { + std::get<3>(m_parsed_lines[i]) = + ((i < skip_start_lines()) || // start rows to skip + (i >= m_parsed_lines.size() - skip_end_lines()) || // end rows to skip + (((i - skip_start_lines()) % 2 == 1) && // skip every second row... + skip_alt_lines()) || // ...if requested + (m_skip_errors && !std::get<1>(m_parsed_lines[i]).empty())); // skip lines with errors + } +} + +uint32_t GncPriceImport::skip_start_lines () { return m_settings.m_skip_start_lines; } +uint32_t GncPriceImport::skip_end_lines () { return m_settings.m_skip_end_lines; } +bool GncPriceImport::skip_alt_lines () { return m_settings.m_skip_alt_lines; } +bool GncPriceImport::skip_err_lines () { return m_skip_errors; } + +void GncPriceImport::separators (std::string separators) +{ + if (file_format() != GncImpFileFormat::CSV) + return; + + m_settings.m_separators = separators; + auto csvtok = dynamic_cast(m_tokenizer.get()); + csvtok->set_separators (separators); + +} +std::string GncPriceImport::separators () { return m_settings.m_separators; } + +void GncPriceImport::settings (const CsvTransSettings& settings) +{ + /* First apply file format as this may recreate the tokenizer */ + file_format (settings.m_file_format); + /* Only then apply the other settings */ + m_settings = settings; + encoding (m_settings.m_encoding); + + if (file_format() == GncImpFileFormat::CSV) + separators (m_settings.m_separators); + else if (file_format() == GncImpFileFormat::FIXED_WIDTH) + { + auto fwtok = dynamic_cast(m_tokenizer.get()); + fwtok->columns (m_settings.m_column_widths); + } + try + { + tokenize(false); + } + catch (...) + { }; + + /* Tokenizing will clear column types, reset them here + * based on the loaded settings. + */ + std::copy_n (settings.m_column_types_price.begin(), + std::min (m_settings.m_column_types_price.size(), settings.m_column_types_price.size()), + m_settings.m_column_types_price.begin()); + +} + +bool GncPriceImport::save_settings () +{ + + if (trans_preset_is_reserved_name (m_settings.m_name)) + return true; + + /* separators are already copied to m_settings in the separators + * function above. However this is not the case for the column + * widths in fw mode, so do this now. + */ + if (file_format() == GncImpFileFormat::FIXED_WIDTH) + { + auto fwtok = dynamic_cast(m_tokenizer.get()); + m_settings.m_column_widths = fwtok->get_columns(); + } + + return m_settings.save(); +} + +void GncPriceImport::settings_name (std::string name) { m_settings.m_name = name; } +std::string GncPriceImport::settings_name () { return m_settings.m_name; } + +/** Loads a file into a GncPriceImport. This is the first function + * that must be called after creating a new GncPriceImport. As long as + * this function didn't run successfully, the importer can't proceed. + * @param filename Name of the file that should be opened + * @exception may throw std::ifstream::failure on any io error + */ +void GncPriceImport::load_file (const std::string& filename) +{ + + /* Get the raw data first and handle an error if one occurs. */ + try + { + m_tokenizer->load_file (filename); + return; + } + catch (std::ifstream::failure& ios_err) + { + // Just log the error and pass it on the call stack for proper handling + PWARN ("Error: %s", ios_err.what()); + throw; + } +} + +/** Splits a file into cells. This requires having an encoding that + * works (see GncPriceImport::convert_encoding). Tokenizing related options + * should be set to the user's selections before calling this + * function. + * Notes: - this function must be called with guessColTypes set to true once + * before calling it with guessColTypes set to false. + * - if guessColTypes is true, all the column types will be set + * GncPricePropType::NONE right now as real guessing isn't implemented yet + * @param guessColTypes true to guess what the types of columns are based on the cell contents + * @exception std::range_error if tokenizing failed + */ +void GncPriceImport::tokenize (bool guessColTypes) +{ + if (!m_tokenizer) + return; + + uint32_t max_cols = 0; + m_tokenizer->tokenize(); + m_parsed_lines.clear(); + for (auto tokenized_line : m_tokenizer->get_tokens()) + { + m_parsed_lines.push_back (std::make_tuple (tokenized_line, std::string(), + std::make_shared(date_format(), currency_format()), + false)); + auto length = tokenized_line.size(); + if (length > max_cols) + max_cols = length; + } + + /* If it failed, generate an error. */ + if (m_parsed_lines.size() == 0) + { + throw (std::range_error ("Tokenizing failed.")); + return; + } + + m_settings.m_column_types_price.resize(max_cols, GncPricePropType::NONE); + + /* Force reinterpretation of already set columns and/or base_account */ + for (uint32_t i = 0; i < m_settings.m_column_types_price.size(); i++) + set_column_type_price (i, m_settings.m_column_types_price[i], true); + + if (guessColTypes) + { + /* Guess column_types based + * on the contents of each column. */ + /* TODO Make it actually guess. */ + } +} + + +struct ErrorListPrice +{ +public: + void add_error (std::string msg); + std::string str(); + bool empty() { return m_error.empty(); } +private: + std::string m_error; +}; + +void ErrorListPrice::add_error (std::string msg) +{ + m_error += "- " + msg + "\n"; +} + +std::string ErrorListPrice::str() +{ + return m_error.substr(0, m_error.size() - 1); +} + + +/* Test for the required minimum number of columns selected and + * the selection is consistent. + * @param An ErrorListPrice object to which all found issues are added. + */ +void GncPriceImport::verify_column_selections (ErrorListPrice& error_msg) +{ + /* Verify if a date column is selected and it's parsable. + */ + if (!check_for_column_type(GncPricePropType::DATE)) + error_msg.add_error( _("Please select a date column.")); + + /* Verify an amount column is selected. + */ + if (!check_for_column_type(GncPricePropType::AMOUNT)) + error_msg.add_error( _("Please select an amount column.")); + + /* Verify an Currency to column is selected. + */ + if (!check_for_column_type(GncPricePropType::CURRENCY_TO)) + error_msg.add_error( _("Please select a Currency to column.")); + + /* Verify at least one from column (symbol_from or currency_from) column is selected. + */ + if (!check_for_column_type(GncPricePropType::SYMBOL_FROM) && + !check_for_column_type(GncPricePropType::CURRENCY_FROM)) + error_msg.add_error( _("Please select a symbol or currency from column.")); +} + + +/* Check whether the chosen settings can successfully parse + * the import data. This will check: + * - there's at least one line selected for import + * - the minimum number of columns is selected + * - the values in the selected columns can be parsed meaningfully. + * @return An empty string if all checks passed or the reason + * verification failed otherwise. + */ +std::string GncPriceImport::verify () +{ + auto newline = std::string(); + auto error_msg = ErrorListPrice(); + + /* Check if the import file did actually contain any information */ + if (m_parsed_lines.size() == 0) + { + error_msg.add_error(_("No valid data found in the selected file. It may be empty or the selected encoding is wrong.")); + return error_msg.str(); + } + + /* Check if at least one line is selected for importing */ + auto skip_alt_offset = m_settings.m_skip_alt_lines ? 1 : 0; + if (m_settings.m_skip_start_lines + m_settings.m_skip_end_lines + skip_alt_offset >= m_parsed_lines.size()) + { + error_msg.add_error(_("No lines are selected for importing. Please reduce the number of lines to skip.")); + return error_msg.str(); + } + + verify_column_selections (error_msg); + + update_skipped_lines (boost::none, boost::none, boost::none, boost::none); + + auto have_line_errors = false; + for (auto line : m_parsed_lines) + { + if (!std::get<3>(line) && !std::get<1>(line).empty()) + { + have_line_errors = true; + break; + } + } + + if (have_line_errors) + error_msg.add_error( _("Not all fields could be parsed. Please correct the issues reported for each line or adjust the lines to skip.")); + + return error_msg.str(); +} + +/** Checks whether the parsed line contains all essential properties. + * @param parsed_line The line we are checking + * @exception std::invalid_argument in an essential property is missing + */ +static void price_properties_verify_essentials (std::vector::iterator& parsed_line) +{ + std::string error_message; + std::shared_ptr price_props; + std::tie(std::ignore, error_message, price_props, std::ignore) = *parsed_line; + + auto price_error = price_props->verify_essentials(); + + error_message.clear(); + if (!price_error.empty()) + { + error_message += price_error; + error_message += "\n"; + } + + if (!error_message.empty()) + throw std::invalid_argument(error_message); +} + +void GncPriceImport::create_price (std::vector::iterator& parsed_line) +{ + StrVec line; + std::string error_message; + std::shared_ptr price_props = nullptr; + bool skip_line = false; + std::tie(line, error_message, price_props, skip_line) = *parsed_line; + + if (skip_line) + return; + + error_message.clear(); + + /* If column parsing was successful, convert price properties into a price. */ + try + { + price_properties_verify_essentials (parsed_line); + + QofBook* book = gnc_get_current_book(); + GNCPriceDB *pdb = gnc_pricedb_get_db (book); + + /* If all went well, add this price to the list. */ + auto price_created = price_props->create_price (book, pdb, m_over_write); +//FIXME Need to look at this + if (price_created) + m_prices_added++; + else + m_prices_duplicated++; + } + catch (const std::invalid_argument& e) + { + error_message = e.what(); + PINFO("User warning: %s", error_message.c_str()); + } +} + + +/** Creates a list of prices from parsed data. The parsed data + * will first be validated. If any errors are found in lines that are marked + * for processing (ie not marked to skip) this function will + * throw an error. + * @param skip_errors true skip over lines with errors + * @exception throws std::invalid_argument if data validation or processing fails. + */ +void GncPriceImport::create_prices () +{ + /* Start with verifying the current data. */ + auto verify_result = verify(); + if (!verify_result.empty()) + throw std::invalid_argument (verify_result); + + m_prices_added = 0; + m_prices_duplicated = 0; + + /* Iterate over all parsed lines */ + for (auto parsed_lines_it = m_parsed_lines.begin(); + parsed_lines_it != m_parsed_lines.end(); + ++parsed_lines_it) + { + /* Skip current line if the user specified so */ + if ((std::get<3>(*parsed_lines_it))) + continue; + + /* Should not throw anymore, otherwise verify needs revision */ + create_price (parsed_lines_it); + } + PINFO("Number of lines is %d, added is %d, duplicates is %d", + (int)m_parsed_lines.size(), m_prices_added, m_prices_duplicated); +} + +bool +GncPriceImport::check_for_column_type (GncPricePropType type) +{ + return (std::find (m_settings.m_column_types_price.begin(), + m_settings.m_column_types_price.end(), type) + != m_settings.m_column_types_price.end()); +} + +/* A helper function intended to be called only from set_column_type_price */ +void GncPriceImport::update_price_props (uint32_t row, uint32_t col, GncPricePropType prop_type) +{ + if (prop_type == GncPricePropType::NONE) + return; /* Only deal with price related properties. */ + + auto price_props = std::make_shared (*(std::get<2>(m_parsed_lines[row])).get()); + auto value = std::string(); + + if (col < std::get<0>(m_parsed_lines[row]).size()) + value = std::get<0>(m_parsed_lines[row]).at(col); + + if (value.empty()) + price_props->reset (prop_type); + else + { + try + { + price_props->set(prop_type, value); + } + catch (const std::exception& e) + { + /* Do nothing, just prevent the exception from escalating up + * However log the error if it happens on a row that's not skipped + */ + if (!std::get<3>(m_parsed_lines[row])) + PINFO("User warning: %s", e.what()); + } + } + /* Store the result */ + std::get<2>(m_parsed_lines[row]) = price_props; +} + +void +GncPriceImport::set_column_type_price (uint32_t position, GncPricePropType type, bool force) +{ + if (position >= m_settings.m_column_types_price.size()) + return; + + auto old_type = m_settings.m_column_types_price[position]; + if ((type == old_type) && !force) + return; /* Nothing to do */ + + // Column types should be unique, so remove any previous occurrence of the new type + std::replace(m_settings.m_column_types_price.begin(), m_settings.m_column_types_price.end(), + type, GncPricePropType::NONE); + + m_settings.m_column_types_price.at (position) = type; + + /* Update the preparsed data */ + for (auto parsed_lines_it = m_parsed_lines.begin(); + parsed_lines_it != m_parsed_lines.end(); + ++parsed_lines_it) + { + /* Reset date and currency formats for each price props object + * to ensure column updates use the most recent one + */ + std::get<2>(*parsed_lines_it)->set_date_format (m_settings.m_date_format); + std::get<2>(*parsed_lines_it)->set_currency_format (m_settings.m_currency_format); + + uint32_t row = parsed_lines_it - m_parsed_lines.begin(); + + /* If the column type actually changed, first reset the property + * represented by the old column type + */ + if (old_type != type) + { + auto old_col = std::get<0>(*parsed_lines_it).size(); // Deliberately out of bounds to trigger a reset! + if ((old_type > GncPricePropType::NONE) + && (old_type <= GncPricePropType::PRICE_PROPS)) + update_price_props (row, old_col, old_type); + } + /* Then set the property represented by the new column type */ + if ((type > GncPricePropType::NONE) + && (type <= GncPricePropType::PRICE_PROPS)) + update_price_props (row, position, type); + + /* Report errors if there are any */ + auto price_errors = std::get<2>(*parsed_lines_it)->errors(); + std::get<1>(*parsed_lines_it) = + price_errors + + (price_errors.empty() ? std::string() : "\n"); + } +} + +std::vector GncPriceImport::column_types_price () +{ + return m_settings.m_column_types_price; +} + diff --git a/gnucash/import-export/csv-imp/gnc-price-import.hpp b/gnucash/import-export/csv-imp/gnc-price-import.hpp new file mode 100644 index 0000000000..0898215b29 --- /dev/null +++ b/gnucash/import-export/csv-imp/gnc-price-import.hpp @@ -0,0 +1,160 @@ +/********************************************************************\ + * gnc-price-import.hpp - import prices from csv files * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License* + * along with this program; if not, contact: * + * * + * Free Software Foundation Voice: +1-617-542-5942 * + * 51 Franklin Street, Fifth Floor Fax: +1-617-542-2652 * + * Boston, MA 02110-1301, USA gnu@gnu.org * +\********************************************************************/ + +/** @file + @brief Class to import prices from CSV or fixed width files + * + gnc-price-import.hpp + @author Copyright (c) 2015 Geert Janssens + @author Copyright (c) 2017 Robert Fewell + */ + +#ifndef GNC_PRICE_IMPORT_HPP +#define GNC_PRICE_IMPORT_HPP + +extern "C" { +#include "config.h" + +} + +#include +#include +#include +#include + +#include "gnc-tokenizer.hpp" +#include "gnc-price-props.hpp" +#include "gnc-csv-trans-settings.hpp" +#include + +/* A set of currency formats that the user sees. */ +extern const int num_currency_formats; +extern const gchar* currency_format_user[]; + +/* A set of date formats that the user sees. */ +extern const int num_date_formats; +extern const gchar* date_format_user[]; + +/** Tuple to hold + * - a tokenized line of input + * - an optional error string + * - a struct to hold user selected properties for a price */ +using parse_line_t = std::tuple, + bool>; +struct ErrorListPrice; + +/** The actual PriceImport class + * It's intended to use in the following sequence of actions: + * - set a file format + * - load a file + * - optionally convert it's encoding + * - parse the file into lines, which in turn are split up in columns + * the result of this step can be queried from tokenizer + * - the user should now map the columns to types, which is stored in column_types + * - last step is convert the mapped columns into a list of transactions + * - this list will then be passed on the the generic importer for further processing */ +class GncPriceImport +{ +public: + // Constructor - Destructor + GncPriceImport(GncImpFileFormat format = GncImpFileFormat::UNKNOWN); + ~GncPriceImport(); + + void file_format(GncImpFileFormat format); + GncImpFileFormat file_format(); + + void over_write (bool over); + bool over_write (); + + void currency_format (int currency_format); + int currency_format (); + + void date_format (int date_format); + int date_format (); + + void encoding (const std::string& encoding); + std::string encoding (); + + void update_skipped_lines (boost::optional start, boost::optional end, + boost::optional alt, boost::optional errors); + uint32_t skip_start_lines (); + uint32_t skip_end_lines (); + bool skip_alt_lines (); + bool skip_err_lines (); + + void separators (std::string separators); + std::string separators (); + + void settings (const CsvTransSettings& settings); + bool save_settings (); + + void settings_name (std::string name); + std::string settings_name (); + + + void load_file (const std::string& filename); + + void tokenize (bool guessColTypes); + + std::string verify(); + + /** This function will attempt to convert all tokenized lines into + * prices using the column types the user has set. + */ + void create_prices (); + bool check_for_column_type (GncPricePropType type); + void set_column_type_price (uint32_t position, GncPricePropType type, bool force = false); + std::vector column_types_price (); + + std::unique_ptr m_tokenizer; /**< Will handle file loading/encoding conversion/splitting into fields */ + std::vector m_parsed_lines; /**< source file parsed into a two-dimensional array of strings. + Per line also holds possible error messages and objects with extracted + price properties. */ + int m_prices_added; + int m_prices_duplicated; + +private: + /** A helper function used by create_prices. It will attempt + * to convert a single tokenized line into a price using + * the column types the user has set. + */ + void create_price (std::vector::iterator& parsed_line); + + void verify_column_selections (ErrorListPrice& error_msg); + + /* Internal helper function to force reparsing of columns subject to format changes */ + void reset_formatted_column (std::vector& col_types); + + /* Two internal helper functions that should only be called from within + * set_column_type_price for consistency (otherwise error messages may not be (re)set) + */ + void update_price_props (uint32_t row, uint32_t col, GncPricePropType prop_type); + + struct CsvTranSettings; + CsvTransSettings m_settings; + bool m_skip_errors; + bool m_over_write; +}; + + +#endif diff --git a/gnucash/import-export/csv-imp/gnc-price-props.hpp b/gnucash/import-export/csv-imp/gnc-price-props.hpp index 6f1189be1a..d1012805f1 100644 --- a/gnucash/import-export/csv-imp/gnc-price-props.hpp +++ b/gnucash/import-export/csv-imp/gnc-price-props.hpp @@ -31,7 +31,7 @@ extern "C" { #endif #include - +#include "gnc-pricedb.h" #include "gnc-commodity.h" }