IntenseFinance/gnucash
3
0
Fork 0
mirror of https://github.com/Gnucash/gnucash.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add code to parse json data returned by our F::Q wrapper

Browse Source 
This code will convert the json data into GncPrice objects and add them
to the pricedb, effectively doing what price-quotes.scm does.

A few notable remarks:
- still requires plenty of cleaning up. This is the first proof of concept
- like the original scm based code, this parser completely ignores  timezone
  information. As it wasn't used before and nobody complained, it may not
  be that important. Or it can be implemented later.
- price-quotes.scm would first check if a price already existed in the pricedb
  and try to update that one instead of adding one (only if the old price's
  type is inferior). However that is redundant as gnc_pricedb_add_price does
  the same check. So I have omitted this extra check from GncQuotes.
- currency quotes can be inverted. I have slightly changed the way to handle
  this. The perl wrapper code will simply set an "inverted" flag in that case,
  but will otherwise not swap currency and commodity as it used to be the case.
  On parsing, the inversion flag will cause the GncNumeric that's parsed from
  the price to be inverted. As it's still a GncNumeric that shouldn't result
  in any loss of precision, while keeping prices in the db always in the default
  currency.
This commit is contained in:
Geert Janssens 2021-02-28 22:36:22 +01:00 committed by John Ralls
parent 5c13da0e59
commit fcbe6cf10c
2 changed files with 259 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
gnucash/gnucash-commands.cpp
View File

@ -54,8 +54,8 @@ static std::string empty_string{};
/* This static indicates the debugging module that this .o belongs to. */
static QofLogModule log_module = GNC_MOD_GUI;
static void
scm_cleanup_and_exit_with_failure (QofSession *session)
static int
cleanup_and_exit_with_failure (QofSession *session)
{
if (session)
{
@ -71,68 +71,15 @@ scm_cleanup_and_exit_with_failure (QofSession *session)
qof_session_destroy (session);
}
qof_event_resume();
gnc_shutdown (1);
return 1;
}
/* scm_boot_guile doesn't expect to return, so call shutdown ourselves here */
static void
scm_add_quotes(void *data, [[maybe_unused]] int argc, [[maybe_unused]] char **argv)
scm_cleanup_and_exit_with_failure (QofSession *session)
{
auto add_quotes_file = static_cast<const std::string*>(data);
gnc_prefs_init ();
qof_event_suspend();
scm_c_eval_string("(debug-set! stack 200000)");
auto mod = scm_c_resolve_module("gnucash price-quotes");
scm_set_current_module(mod);
auto add_quotes = scm_c_eval_string("gnc:book-add-quotes");
auto session = gnc_get_current_session();
if (!session)
scm_cleanup_and_exit_with_failure (session);
qof_session_begin(session, add_quotes_file->c_str(), SESSION_NORMAL_OPEN);
if (qof_session_get_error(session) != ERR_BACKEND_NO_ERR)
scm_cleanup_and_exit_with_failure (session);
qof_session_load(session, NULL);
if (qof_session_get_error(session) != ERR_BACKEND_NO_ERR)
scm_cleanup_and_exit_with_failure (session);
GncQuotes quotes (qof_session_get_book(session));
if (quotes.cmd_result() == 0)
{
std::cout << bl::format (bl::translate ("Found Finance::Quote version {1}.")) % quotes.version() << std::endl;
auto quote_sources = quotes.sources_as_glist();
gnc_quote_source_set_fq_installed (quotes.version().c_str(), quote_sources);
g_list_free_full (quote_sources, g_free);
}
else
{
std::cerr << bl::translate ("No quotes retrieved. Finance::Quote isn't "
"installed properly.") << "\n";
std::cerr << bl::translate ("Error message:") << std::endl;
std::cerr << quotes.error_msg() << std::endl;
}
auto scm_book = gnc_book_to_scm(qof_session_get_book(session));
auto scm_result = scm_call_2(add_quotes, SCM_BOOL_F, scm_book);
qof_session_save(session, NULL);
if (qof_session_get_error(session) != ERR_BACKEND_NO_ERR)
scm_cleanup_and_exit_with_failure (session);
qof_session_destroy(session);
if (!scm_is_true(scm_result))
{
PERR ("Failed to add quotes to %s.", add_quotes_file->c_str());
scm_cleanup_and_exit_with_failure (session);
}
qof_event_resume();
gnc_shutdown(0);
return;
cleanup_and_exit_with_failure (session);
gnc_shutdown (1);
}
static void
@ -379,9 +326,48 @@ Gnucash::quotes_info (void)
int
Gnucash::add_quotes (const bo_str& uri)
{
if (uri && !uri->empty())
scm_boot_guile (0, nullptr, scm_add_quotes, (void *)&(*uri));
gnc_prefs_init ();
qof_event_suspend();
auto session = gnc_get_current_session();
if (!session)
return 1;
qof_session_begin(session, uri->c_str(), SESSION_NORMAL_OPEN);
if (qof_session_get_error(session) != ERR_BACKEND_NO_ERR)
cleanup_and_exit_with_failure (session);
qof_session_load(session, NULL);
if (qof_session_get_error(session) != ERR_BACKEND_NO_ERR)
cleanup_and_exit_with_failure (session);
GncQuotes quotes (qof_session_get_book(session));
if (quotes.cmd_result() == 0)
{
std::cout << bl::format (bl::translate ("Found Finance::Quote version {1}.")) % quotes.version() << std::endl;
auto quote_sources = quotes.sources_as_glist();
gnc_quote_source_set_fq_installed (quotes.version().c_str(), quote_sources);
g_list_free_full (quote_sources, g_free);
}
else
{
std::cerr << bl::translate ("No quotes retrieved. Finance::Quote isn't "
"installed properly.") << "\n";
std::cerr << bl::translate ("Error message:") << std::endl;
std::cerr << quotes.error_msg() << std::endl;
}
quotes.fetch_all ();
qof_session_save(session, NULL);
if (qof_session_get_error(session) != ERR_BACKEND_NO_ERR)
cleanup_and_exit_with_failure (session);
qof_session_destroy(session);
if (quotes.cmd_result() != 0)
std::cerr << bl::format (bl::translate ("Failed to add quotes to {1}.")) % *uri << "\n";
qof_event_resume();
return 0;
}

222
libgnucash/app-utils/gnc-quotes.cpp
View File

@ -27,6 +27,7 @@
#include <string>
#include <iostream>
#include <sstream>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include <boost/process.hpp>
#include <boost/property_tree/ptree.hpp>
@ -36,6 +37,8 @@
#include <boost/asio.hpp>
#include <glib.h>
#include "gnc-commodity.hpp"
#include <gnc-datetime.hpp>
#include <gnc-numeric.hpp>
#include "gnc-quotes.hpp"
extern "C" {
@ -48,6 +51,7 @@ extern "C" {
}
namespace bp = boost::process;
namespace bfs = boost::filesystem;
namespace bpt = boost::property_tree;
namespace bio = boost::iostreams;
@ -78,8 +82,12 @@ private:
// - one with the contents of stdout
// - one with the contents of stderr
// Will also set m_cmd_result
CmdOutput run_cmd (std::string cmd_name, StrVec args, StrVec input_vec);
template <typename BufferT> CmdOutput run_cmd (const bfs::path &cmd_name, StrVec args, BufferT input);
void parse_quotes (const std::string &quotes);
CommVec m_comm_vec;
std::string m_version;
QuoteSources m_sources;
int m_cmd_result;
@ -140,11 +148,13 @@ GncQuotesImpl::sources_as_glist()
void
GncQuotesImpl::fetch (const CommVec& commodities)
{
m_comm_vec = commodities; // Store for later use
auto dflt_curr = gnc_default_currency();
bpt::ptree pt, pt_child;
pt.put ("defaultcurrency", gnc_commodity_get_mnemonic (dflt_curr));
std::for_each (commodities.cbegin(), commodities.cend(),
std::for_each (m_comm_vec.cbegin(), m_comm_vec.cend(),
[&pt, &dflt_curr] (auto comm)
{
auto comm_mnemonic = gnc_commodity_get_mnemonic (comm);
@ -152,7 +162,7 @@ GncQuotesImpl::fetch (const CommVec& commodities)
if (gnc_commodity_is_currency (comm))
{
if (gnc_commodity_equiv(comm, dflt_curr) ||
(!comm_mnemonic || (strcmp (comm_mnemonic, "XXX") == 0)))
(!comm_mnemonic || (strcmp (comm_mnemonic, "XXX") == 0)))
return;
}
else
@ -165,7 +175,30 @@ GncQuotesImpl::fetch (const CommVec& commodities)
std::ostringstream result;
bpt::write_json(result, pt);
std::cerr << "GncQuotes fetch_all - resulting json object\n" << result.str() << std::endl;
//std::cerr << "GncQuotes fetch_all - resulting json object\n" << result.str() << std::endl;
auto perl_executable = bp::search_path("perl");
auto fq_wrapper = std::string(gnc_path_get_bindir()) + "/finance-quote-wrapper";
StrVec args { "-w", fq_wrapper };
auto cmd_out = run_cmd (perl_executable.string(), args, result.str());
if (m_cmd_result == 0)
{
std::string resultstr;
for (auto line : cmd_out.first)
resultstr.append(std::move(line) + "\n");
parse_quotes (resultstr);
}
else
for (auto line : cmd_out.second)
m_error_msg.append(std::move(line) + "\n");
for (auto line : cmd_out.first)
std::cerr << "Output line retrieved from wrapper:\n" << line << std::endl;
for (auto line : cmd_out.second)
std::cerr << "Error line retrieved from wrapper:\n" << line << std::endl;
}
@ -186,22 +219,27 @@ format_quotes (const std::vector<gnc_commodity*>)
}
CmdOutput
GncQuotesImpl::run_cmd (std::string cmd_name, StrVec args, StrVec input_vec)
template <typename BufferT> CmdOutput
GncQuotesImpl::run_cmd (const bfs::path &cmd_name, StrVec args, BufferT input)
{
StrVec out_vec, err_vec;
auto av_key = gnc_prefs_get_string ("general.finance-quote", "alphavantage-api-key");
if (!av_key)
std::cerr << "No AlphaVantage API key set, currency quotes and other AlphaVantage based quotes won't work." << std::endl;
try
{
std::future<std::vector<char> > out_buf, err_buf;
boost::asio::io_service svc;
auto input_buf = bp::buffer (input_vec);
auto input_buf = bp::buffer (input);
bp::child process (cmd_name, args,
bp::std_out > out_buf,
bp::std_err > err_buf,
bp::std_in < input_buf,
svc);
bp::std_out > out_buf,
bp::std_err > err_buf,
bp::std_in < input_buf,
bp::env["ALPHAVANTAGE_API_KEY"]= (av_key ? av_key : ""),
svc);
svc.run();
process.wait();
@ -233,6 +271,168 @@ GncQuotesImpl::run_cmd (std::string cmd_name, StrVec args, StrVec input_vec)
return CmdOutput (std::move(out_vec), std::move(err_vec));
}
void
GncQuotesImpl::parse_quotes (const std::string &quotes_str)
{
bpt::ptree pt;
std::istringstream ss {quotes_str};
try
{
bpt::read_json (ss, pt);
}
catch (bpt::json_parser_error &e) {
m_cmd_result = -1;
m_error_msg = m_error_msg +
"Failed to parse quotes results." + "\n" +
"Error message:" + "\n" +
e.what() + "\n";
}
catch (...) {
m_cmd_result = -1;
m_error_msg = m_error_msg +
"Failed to parse quotes results." + "\n";
}
auto book = m_book;
auto dflt_curr = gnc_default_currency();
auto pricedb = gnc_pricedb_get_db (m_book);
std::for_each(m_comm_vec.begin(), m_comm_vec.end(),
[this, &pt, &dflt_curr, &pricedb] (gnc_commodity *comm)
{
auto comm_ns = gnc_commodity_get_namespace (comm);
auto comm_mnemonic = gnc_commodity_get_mnemonic (comm);
if (gnc_commodity_equiv(comm, dflt_curr) ||
(!comm_mnemonic || (strcmp (comm_mnemonic, "XXX") == 0)))
return;
if (pt.find (comm_mnemonic) == pt.not_found())
{
std::cerr << "Skipped " << comm_ns << ":" << comm_mnemonic << " - Finance::Quote didn't return any data.\n";
return;
}
std::string key = comm_mnemonic;
boost::optional<bool> success = pt.get_optional<bool> (key + ".success");
std::string price_type = "last";
boost::optional<std::string> price_str = pt.get_optional<std::string> (key + "." + price_type);
if (!price_str)
{
price_type = "nav";
price_str = pt.get_optional<std::string> (key + "." + price_type);
}
if (!price_str)
{
price_type = "price";
price_str = pt.get_optional<std::string> (key + "." + price_type);
/* guile wrapper used "unknown" as price type when "price" was found,
* reproducing here to keep same result for users in the pricedb */
price_type = "unknown";
}
boost::optional<bool> inverted_tmp = pt.get_optional<bool> (key + ".inverted");
bool inverted = inverted_tmp ? *inverted_tmp : false;
boost::optional<std::string> date_str = pt.get_optional<std::string> (key + ".date");
boost::optional<std::string> time_str = pt.get_optional<std::string> (key + ".time");
boost::optional<std::string> currency_str = pt.get_optional<std::string> (key + ".currency");
std::cout << "Commodity: " << comm_mnemonic << "\n";
std::cout << " Date: " << (date_str ? *date_str : "missing") << "\n";
std::cout << " Time: " << (time_str ? *time_str : "missing") << "\n";
std::cout << " Currency: " << (currency_str ? *currency_str : "missing") << "\n";
std::cout << " Price: " << (price_str ? *price_str : "missing") << "\n";
std::cout << " Inverted: " << (inverted ? "yes" : "no") << "\n\n";
if (!success || !*success)
{
boost::optional<std::string> errmsg = pt.get_optional<std::string> (key + ".errormsg");
std::cerr << "Skipped " << comm_ns << ":" << comm_mnemonic << " - Finance::Quote returned fetch failure.\n";
std::cerr << "Reason: " << (errmsg ? *errmsg : "unknown") << "\n";
return;
}
if (!price_str)
{
std::cerr << "Skipped " << comm_ns << ":" << comm_mnemonic << " - Finance::Quote didn't return a valid price\n";
return;
}
GncNumeric price;
try
{
price = GncNumeric { *price_str };
}
catch (...)
{
std::cerr << "Skipped " << comm_ns << ":" << comm_mnemonic << " - failed to parse returned price '" << *price_str << "'\n";
return;
}
if (inverted)
price = price.inv();
if (!currency_str)
{
std::cerr << "Skipped " << comm_ns << ":" << comm_mnemonic << " - Finance::Quote didn't return a currency\n";
return;
}
boost::to_upper (*currency_str);
auto commodity_table = gnc_commodity_table_get_table (m_book);
auto currency = gnc_commodity_table_lookup (commodity_table, "ISO4217", currency_str->c_str());
if (!currency)
{
std::cerr << "Skipped " << comm_ns << ":" << comm_mnemonic << " - failed to parse returned currency '" << *currency_str << "'\n";
return;
}
std::string iso_date_str = GncDate().format ("%Y-%m-%d");
if (date_str)
{
// Returned date is always in MM/DD/YYYY format according to F::Q man page, transform it to simplify conversion to GncDateTime
auto date_tmp = *date_str;
iso_date_str = date_tmp.substr (6, 4) + "-" + date_tmp.substr (0, 2) + "-" + date_tmp.substr (3, 2);
}
else
std::cerr << "Info: no date was returned for " << comm_ns << ":" << comm_mnemonic << " - will use today\n";
iso_date_str += " " + (time_str ? *time_str : "12:00:00");
auto can_convert = true;
try
{
GncDateTime testdt {iso_date_str};
}
catch (...)
{
std::cerr << "Warning: failed to parse quote date and time '" << iso_date_str << "' for " << comm_ns << ":" << comm_mnemonic << " - will use today\n";
return;
}
/* Bit of an odd construct: GncDateTimes can't be copied,
which makes it impossible to first create a temporary GncDateTime
based on whether the string is parsable and then assign that temporary
to our final GncDateTime. The creation has to happen in one go, so
below construct will pass a different constructor argument based on
whether a test conversion worked or not.
*/
GncDateTime quotedt {can_convert ? iso_date_str : GncDateTime()};
auto gnc_price = gnc_price_create (m_book);
gnc_price_begin_edit (gnc_price);
gnc_price_set_commodity (gnc_price, comm);
gnc_price_set_currency (gnc_price, currency);
gnc_price_set_time64 (gnc_price, static_cast<time64> (quotedt));
gnc_price_set_source (gnc_price, PRICE_SOURCE_FQ);
gnc_price_set_typestr (gnc_price, price_type.c_str());
gnc_price_set_value (gnc_price, price);
gnc_pricedb_add_price (pricedb, gnc_price);
gnc_price_commit_edit (gnc_price);
gnc_price_unref (gnc_price);
});
}
/********************************************************************
* gnc_quotes_get_quotable_commodities