Add dummy tokenizer to be used when file format isn't known yet

src/import-export/csv-imp/CMakeLists.txt

@@ -14,6 +14,7 @@ SET(csv_import_SOURCES
   gnc-csv-gnumeric-popup.c
   gnc-csv-tokenizer.cpp
   gnc-csv-trans-settings.c
+  gnc-dummy-tokenizer.cpp
   gnc-fw-tokenizer.cpp
   gnc-tokenizer.cpp
   ${CMAKE_SOURCE_DIR}/lib/stf/stf-parse.c
@@ -37,6 +38,7 @@ SET(csv_import_noinst_HEADERS
   gnc-csv-gnumeric-popup.h
   gnc-csv-tokenizer.hpp
   gnc-csv-trans-settings.h
+  gnc-dummy-tokenizer.hpp
   gnc-fw-tokenizer.hpp
   gnc-tokenizer.hpp
   ${CMAKE_SOURCE_DIR}/lib/stf/stf-parse.h

src/import-export/csv-imp/Makefile.am

@@ -14,6 +14,7 @@ libgncmod_csv_import_la_SOURCES = \
   gnc-csv-model.c \
   gnc-csv-tokenizer.cpp \
   gnc-csv-gnumeric-popup.c \
+  gnc-dummy-tokenizer.cpp \
   gnc-fw-tokenizer.cpp \
   gnc-tokenizer.cpp \
   gnc-csv-trans-settings.c
@@ -29,6 +30,7 @@ noinst_HEADERS = \
   gnc-csv-model.h \
   gnc-csv-tokenizer.hpp \
   gnc-csv-gnumeric-popup.h \
+  gnc-dummy-tokenizer.hpp \
   gnc-fw-tokenizer.hpp \
   gnc-tokenizer.hpp \
   gnc-csv-trans-settings.h

src/import-export/csv-imp/gnc-dummy-tokenizer.cpp

@@ -0,0 +1,31 @@
+#include "gnc-dummy-tokenizer.hpp"
+
+#include <iostream>
+#include <fstream>      // fstream
+#include <vector>
+#include <string>
+#include <algorithm>    // copy
+#include <iterator>     // ostream_operator
+
+#include <boost/locale.hpp>
+
+
+int GncDummyTokenizer::tokenize()
+{
+    std::vector<std::string> vec;
+    std::string line;
+
+    tokenized_contents.clear();
+    std::istringstream in_stream(utf8_contents);
+
+    while (std::getline (in_stream, line))
+    {
+        vec.push_back (line);
+        tokenized_contents.push_back(vec);
+
+        line.clear();
+        vec.clear();
+    }
+
+    return 0;
+}

src/import-export/csv-imp/gnc-dummy-tokenizer.hpp

@@ -0,0 +1,62 @@
+/********************************************************************\
+ * gnc-dummy-tokenizer.hpp - takes a file and converts it into a       *
+ *                        two-dimensional vector of strings (table) *
+ *                        splitting the contents on fixed width     *
+ *                        positions                                 *
+ *                                                                  *
+ * This program is free software; you can redistribute it and/or    *
+ * modify it under the terms of the GNU General Public License as   *
+ * published by the Free Software Foundation; either version 2 of   *
+ * the License, or (at your option) any later version.              *
+ *                                                                  *
+ * This program is distributed in the hope that it will be useful,  *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of   *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the    *
+ * GNU General Public License for more details.                     *
+ *                                                                  *
+ * You should have received a copy of the GNU General Public License*
+ * along with this program; if not, contact:                        *
+ *                                                                  *
+ * Free Software Foundation           Voice:  +1-617-542-5942       *
+ * 51 Franklin Street, Fifth Floor    Fax:    +1-617-542-2652       *
+ * Boston, MA  02110-1301,  USA       gnu@gnu.org                   *
+\********************************************************************/
+
+/** @file
+     @brief Dummy converter class to convert a file
+     into vector of string vectors. Each string vector has only one element,
+     the contents of one line of the file.
+     This is just a dummy that can be used as long as the file format isn't
+     specified yet by the user.
+     *
+     gnc-dummy-tokenizer.hpp
+     @author Copyright (c) 2016 Geert Janssens <geert@kobaltwit.be>
+ */
+
+#ifndef GNC_DUMMY_TOKENIZER_HPP
+#define GNC_DUMMY_TOKENIZER_HPP
+
+extern "C" {
+#include "config.h"
+}
+
+#include <iostream>
+#include <fstream>      // fstream
+#include <vector>
+#include <string>
+#include "gnc-tokenizer.hpp"
+
+class GncDummyTokenizer : public GncTokenizer
+{
+public:
+    GncDummyTokenizer() = default;                                 // default constructor
+    GncDummyTokenizer(const GncDummyTokenizer&) = default;            // copy constructor
+    GncDummyTokenizer& operator=(const GncDummyTokenizer&) = default; // copy assignment
+    GncDummyTokenizer(GncDummyTokenizer&&) = default;                 // move constructor
+    GncDummyTokenizer& operator=(GncDummyTokenizer&&) = default;      // move assignment
+    ~GncDummyTokenizer() = default;                                // destructor
+
+    int  tokenize() override;
+};
+
+#endif

src/import-export/csv-imp/gnc-tokenizer.cpp

@@ -1,5 +1,6 @@
 #include "gnc-tokenizer.hpp"
 #include "gnc-csv-tokenizer.hpp"
+#include "gnc-dummy-tokenizer.hpp"
 #include "gnc-fw-tokenizer.hpp"
 
 #include <iostream>
@@ -28,6 +29,7 @@ std::unique_ptr<GncTokenizer> GncTokenizerFactory(GncImpFileFormat fmt)
         tok.reset(new GncFwTokenizer());
         break;
     default:
+        tok.reset(new GncDummyTokenizer());
         break;
     }