Implement unicode conversion using Windows native functions (#1590)

* Implement unicode conversion using Windows native functions

* NOCPPLINT

* Fixed deprecated c++ api usage in tests

* Moved impl to cpp

* Moved Unicode utils to Plugin API

* Added missed include for Windows

* Fixes for unit tests; CentOS fixes

* Fixed Windows compilation

* Fixed unit tests on Unix

* Fixed unix 2
This commit is contained in:
Ilya Lavrenov 2020-08-06 12:01:34 +03:00 committed by GitHub
parent ea34f04028
commit 6a5993fb36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 152 additions and 122 deletions

View File

@ -14,12 +14,10 @@
#include <type_traits>
#include "ie_common.h"
#include "ie_unicode.hpp"
#include "ie_so_loader.h"
#include "details/ie_exception.hpp"
#include "details/ie_no_release.hpp"
#include "details/ie_irelease.hpp"
#include "details/os/os_filesystem.hpp"
namespace InferenceEngine {
namespace details {
@ -79,6 +77,13 @@ private:
template <class T>
class SOCreatorTrait {};
/**
* @brief Enables only `char` or `wchar_t` template specializations
* @tparam C A char type
*/
template <typename C>
using enableIfSupportedChar = typename std::enable_if<(std::is_same<C, char>::value || std::is_same<C, wchar_t>::value)>::type;
/**
* @brief This class instantiate object using shared library
* @tparam T An type of object SOPointer can hold
@ -211,6 +216,9 @@ protected:
* @return A created object
*/
template <class T>
inline std::shared_ptr<T> make_so_pointer(const file_name_t& name) = delete;
inline std::shared_ptr<T> make_so_pointer(const std::string & name) = delete;
template <class T>
inline std::shared_ptr<T> make_so_pointer(const std::wstring & name) = delete;
} // namespace InferenceEngine

View File

@ -1,49 +0,0 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief This is a header file with functions related to filesystem operations
*
* @file os_filesystem.hpp
*/
#pragma once
#include "ie_api.h"
#ifdef ENABLE_UNICODE_PATH_SUPPORT
# include <codecvt>
#endif
#include <locale>
#include <string>
namespace InferenceEngine {
namespace details {
template<typename C>
using enableIfSupportedChar = typename std::enable_if<(std::is_same<C, char>::value || std::is_same<C, wchar_t>::value)>::type;
#ifdef ENABLE_UNICODE_PATH_SUPPORT
/**
* @brief Conversion from wide character string to a single-byte chain.
*/
inline const std::string wStringtoMBCSstringChar(const std::wstring& wstr) {
std::wstring_convert<std::codecvt_utf8<wchar_t>> wstring_decoder;
return wstring_decoder.to_bytes(wstr);
}
/**
* @brief Conversion from single-byte chain to wide character string.
*/
inline const std::wstring multiByteCharToWString(const char* str) {
std::wstring_convert<std::codecvt_utf8<wchar_t>> wstring_encoder;
std::wstring result = wstring_encoder.from_bytes(str);
return result;
}
#endif // ENABLE_UNICODE_PATH_SUPPORT
} // namespace details
} // namespace InferenceEngine

View File

@ -90,10 +90,7 @@
#ifndef ENABLE_UNICODE_PATH_SUPPORT
# ifdef _WIN32
# ifdef __INTEL_COMPILER
# define ENABLE_UNICODE_PATH_SUPPORT
# endif
# if defined _MSC_VER && defined _MSVC_LANG && _MSVC_LANG < 201703L
# if defined __INTEL_COMPILER || defined _MSC_VER
# define ENABLE_UNICODE_PATH_SUPPORT
# endif
# elif defined(__GNUC__) && (__GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ > 2)) || defined(__clang__)

View File

@ -18,7 +18,6 @@
#include "ie_extension.h"
#include "ie_remote_context.hpp"
#include "cpp/ie_executable_network.hpp"
#include "details/os/os_filesystem.hpp"
namespace InferenceEngine {
@ -61,9 +60,7 @@ public:
* ONNX models with data files are not supported
* @return CNNNetwork
*/
CNNNetwork ReadNetwork(const std::wstring& modelPath, const std::wstring& binPath = {}) const {
return ReadNetwork(details::wStringtoMBCSstringChar(modelPath), details::wStringtoMBCSstringChar(binPath));
}
CNNNetwork ReadNetwork(const std::wstring& modelPath, const std::wstring& binPath = {}) const;
#endif
/**

View File

@ -14,7 +14,6 @@
#include <string>
#include <vector>
#include "ie_unicode.hpp"
#include "ie_iextension.h"
#include "details/ie_so_pointer.hpp"
@ -46,7 +45,9 @@ public:
*
* @param name Full or relative path to extension library
*/
explicit Extension(const file_name_t& name): actual(name) {}
template <typename C,
typename = details::enableIfSupportedChar<C>>
explicit Extension(const std::basic_string<C>& name): actual(name) {}
/**
* @brief Gets the extension version information
@ -101,18 +102,27 @@ protected:
/**
* @brief A SOPointer instance to the loaded templated object
*/
InferenceEngine::details::SOPointer<IExtension> actual;
details::SOPointer<IExtension> actual;
};
/**
* @brief Creates a special shared_pointer wrapper for the given type from a specific shared module
*
* @param name Name of the shared library file
* @param name A std::string name of the shared library file
* @return shared_pointer A wrapper for the given type from a specific shared module
*/
template <>
inline std::shared_ptr<IExtension> make_so_pointer(const file_name_t& name) {
inline std::shared_ptr<IExtension> make_so_pointer(const std::string& name) {
return std::make_shared<Extension>(name);
}
#ifdef ENABLE_UNICODE_PATH_SUPPORT
template <>
inline std::shared_ptr<IExtension> make_so_pointer(const std::wstring& name) {
return std::make_shared<Extension>(name);
}
#endif
} // namespace InferenceEngine

View File

@ -27,8 +27,10 @@ typedef std::string file_name_t;
namespace InferenceEngine {
/**
* @deprecated Use OS-native conversion utilities
* @brief Conversion from possibly-wide character string to a single-byte chain.
*/
INFERENCE_ENGINE_DEPRECATED("Use OS-native conversion utilities")
inline std::string fileNameToString(const file_name_t& str) {
#ifdef UNICODE
size_t maxlen = (str.length() + 1) * sizeof(wchar_t) / sizeof(char);
@ -43,8 +45,10 @@ inline std::string fileNameToString(const file_name_t& str) {
}
/**
* @deprecated Use OS-native conversion utilities
* @brief Conversion from single-byte character string to a possibly-wide one
*/
INFERENCE_ENGINE_DEPRECATED("Use OS-native conversion utilities")
inline file_name_t stringToFileName(const std::string& str) {
#ifdef UNICODE
size_t maxlen = str.length() + 1;

View File

@ -8,7 +8,6 @@
#include <samples/common.hpp>
#include <inference_engine.hpp>
#include <details/os/os_filesystem.hpp>
#include <samples/ocv_common.hpp>
#include <samples/classification_results.h>
@ -55,8 +54,16 @@ cv::Mat imreadW(std::wstring input_image_path) {
return image;
}
std::string simpleConvert(const std::wstring & wstr) {
std::string str;
for (auto && wc : wstr)
str += static_cast<char>(wc);
return str;
}
int wmain(int argc, wchar_t *argv[]) {
#else
int main(int argc, char *argv[]) {
#endif
try {
@ -69,7 +76,7 @@ int main(int argc, char *argv[]) {
const file_name_t input_model{argv[1]};
const file_name_t input_image_path{argv[2]};
#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
const std::string device_name = InferenceEngine::details::wStringtoMBCSstringChar(argv[3]);
const std::string device_name = simpleConvert(argv[3]);
#else
const std::string device_name{argv[3]};
#endif

View File

@ -18,6 +18,8 @@ list(REMOVE_ITEM LIBRARY_SRC ${IE_STATIC_DEPENDENT_FILES})
set(IE_BASE_SOURCE_FILES
${CMAKE_CURRENT_SOURCE_DIR}/cnn_network_ngraph_impl.cpp
# will be merged with file_utils.cpp after IE dependency on legacy is removed
${CMAKE_CURRENT_SOURCE_DIR}/ie_unicode.cpp
${CMAKE_CURRENT_SOURCE_DIR}/generic_ie.cpp
${CMAKE_CURRENT_SOURCE_DIR}/blob_factory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ie_blob_common.cpp
@ -152,8 +154,8 @@ add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}_obj)
# Create shared library file from object library
add_library(${TARGET_NAME} SHARED
$<TARGET_OBJECTS:${TARGET_NAME}_obj>
${IE_STATIC_DEPENDENT_FILES})
${IE_STATIC_DEPENDENT_FILES}
$<TARGET_OBJECTS:${TARGET_NAME}_obj>)
set_ie_threading_interface_for(${TARGET_NAME})

View File

@ -2,20 +2,18 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <file_utils.h>
#include <cstring>
#include <fstream>
#include <string>
#include "details/ie_exception.hpp"
#ifdef __MACH__
#include <mach/clock.h>
#include <mach/mach.h>
#endif
// for PATH_MAX
#include <file_utils.h>
#include <details/ie_exception.hpp>
#ifndef _WIN32
# include <limits.h>
# include <unistd.h>
@ -24,18 +22,9 @@
# include <Windows.h>
#endif
#include "details/ie_so_pointer.hpp"
#include "details/os/os_filesystem.hpp"
#if defined(WIN32) || defined(WIN64)
// Copied from linux libc sys/stat.h:
#define S_ISREG(m) (((m)&S_IFMT) == S_IFREG)
#define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR)
#endif
long long FileUtils::fileSize(const char* charfilepath) {
#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
std::wstring widefilename = InferenceEngine::details::multiByteCharToWString(charfilepath);
std::wstring widefilename = FileUtils::multiByteCharToWString(charfilepath);
const wchar_t* fileName = widefilename.c_str();
#else
const char* fileName = charfilepath;
@ -103,7 +92,7 @@ std::wstring getIELibraryPathW() {
GetModuleFileNameW(hm, (LPWSTR)ie_library_path, sizeof(ie_library_path));
return getPathName(std::wstring(ie_library_path));
#else
return details::multiByteCharToWString(getIELibraryPathA().c_str());
return ::FileUtils::multiByteCharToWString(getIELibraryPathA().c_str());
#endif
}
@ -111,7 +100,7 @@ std::wstring getIELibraryPathW() {
std::string getIELibraryPath() {
#ifdef ENABLE_UNICODE_PATH_SUPPORT
return details::wStringtoMBCSstringChar(getIELibraryPathW());
return FileUtils::wStringtoMBCSstringChar(getIELibraryPathW());
#else
return getIELibraryPathA();
#endif

View File

@ -355,9 +355,7 @@ public:
cppPlugin.SetConfig(desc.defaultConfig);
for (auto&& extensionLocation : desc.listOfExtentions) {
// TODO: fix once InferenceEngine::Extension can accept FileUtils::FilePath
// currently, extensions cannot be loaded using wide path
cppPlugin.AddExtension(make_so_pointer<IExtension>(FileUtils::fromFilePath(extensionLocation)));
cppPlugin.AddExtension(make_so_pointer<IExtension>(extensionLocation));
}
}
@ -551,6 +549,15 @@ std::map<std::string, Version> Core::GetVersions(const std::string& deviceName)
return versions;
}
#ifdef ENABLE_UNICODE_PATH_SUPPORT
CNNNetwork Core::ReadNetwork(const std::wstring& modelPath, const std::wstring& binPath) const {
return ReadNetwork(FileUtils::wStringtoMBCSstringChar(modelPath),
FileUtils::wStringtoMBCSstringChar(binPath));
}
#endif
CNNNetwork Core::ReadNetwork(const std::string& modelPath, const std::string& binPath) const {
return _impl->ReadNetwork(modelPath, binPath);
}

View File

@ -162,7 +162,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath, const std::string&
// Fix unicode name
#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
std::wstring model_path = InferenceEngine::details::multiByteCharToWString(modelPath.c_str());
std::wstring model_path = FileUtils::multiByteCharToWString(modelPath.c_str());
#else
std::string model_path = modelPath;
#endif
@ -197,7 +197,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath, const std::string&
if (!bPath.empty()) {
// Open weights file
#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
std::wstring weights_path = InferenceEngine::details::multiByteCharToWString(bPath.c_str());
std::wstring weights_path = FileUtils::multiByteCharToWString(bPath.c_str());
#else
std::string weights_path = bPath;
#endif

View File

@ -0,0 +1,46 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <string>
#include <file_utils.h>
#ifndef _WIN32
# ifdef ENABLE_UNICODE_PATH_SUPPORT
# include <locale>
# include <codecvt>
# endif
#else
# include <Windows.h>
#endif
#ifdef ENABLE_UNICODE_PATH_SUPPORT
std::string FileUtils::wStringtoMBCSstringChar(const std::wstring& wstr) {
#ifdef _WIN32
int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL); // NOLINT
std::string strTo(size_needed, 0);
WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0], size_needed, NULL, NULL); // NOLINT
return strTo;
#else
std::wstring_convert<std::codecvt_utf8<wchar_t>> wstring_decoder;
return wstring_decoder.to_bytes(wstr);
#endif
}
std::wstring FileUtils::multiByteCharToWString(const char* str) {
#ifdef _WIN32
int strSize = static_cast<int>(std::strlen(str));
int size_needed = MultiByteToWideChar(CP_UTF8, 0, str, strSize, NULL, 0);
std::wstring wstrTo(size_needed, 0);
MultiByteToWideChar(CP_UTF8, 0, str, strSize, &wstrTo[0], size_needed);
return wstrTo;
#else
std::wstring_convert<std::codecvt_utf8<wchar_t>> wstring_encoder;
std::wstring result = wstring_encoder.from_bytes(str);
return result;
#endif
}
#endif // ENABLE_UNICODE_PATH_SUPPORT

View File

@ -5,8 +5,8 @@
#include <dlfcn.h>
#include "details/ie_exception.hpp"
#include "details/os/os_filesystem.hpp"
#include "details/ie_so_loader.h"
#include "file_utils.h"
namespace InferenceEngine {
namespace details {
@ -24,7 +24,7 @@ public:
}
#ifdef ENABLE_UNICODE_PATH_SUPPORT
explicit Impl(const wchar_t* pluginName) : Impl(wStringtoMBCSstringChar(pluginName).c_str()) {
explicit Impl(const wchar_t* pluginName) : Impl(FileUtils::wStringtoMBCSstringChar(pluginName).c_str()) {
}
#endif // ENABLE_UNICODE_PATH_SUPPORT

View File

@ -3,8 +3,8 @@
//
#include "details/ie_exception.hpp"
#include "details/os/os_filesystem.hpp"
#include "details/ie_so_loader.h"
#include "file_utils.h"
#include <direct.h>
#include <windows.h>
@ -36,7 +36,7 @@ public:
shared_object = LoadLibraryW(pluginName);
if (!shared_object) {
char cwd[1024];
THROW_IE_EXCEPTION << "Cannot load library '" << details::wStringtoMBCSstringChar(std::wstring(pluginName)) << "': " << GetLastError()
THROW_IE_EXCEPTION << "Cannot load library '" << FileUtils::wStringtoMBCSstringChar(std::wstring(pluginName)) << "': " << GetLastError()
<< " from cwd: " << _getcwd(cwd, sizeof(cwd));
}
}

View File

@ -9,7 +9,6 @@
#include <ie_icnn_network.hpp>
#include <cnn_network_impl.hpp>
#include <file_utils.h>
#include <deque>
#include <functional>
#include <string>

View File

@ -19,8 +19,6 @@
#include "details/caseless.hpp"
#include "details/ie_cnn_network_tools.h"
#include "details/os/os_filesystem.hpp"
#include "file_utils.h"
#include "graph_tools.hpp"
#include "net_pass.h"
#include "precision_utils.h"

View File

@ -3,7 +3,7 @@
//
/**
* @brief Basic function to work with file system and UNICDE symbols
* @brief Basic function to work with file system and UNICODE symbols
* @file file_utils.h
*/
@ -14,11 +14,28 @@
#include <cstring>
#include "ie_api.h"
#include "ie_unicode.hpp"
#include "details/os/os_filesystem.hpp"
#include "details/ie_so_pointer.hpp"
namespace FileUtils {
#ifdef ENABLE_UNICODE_PATH_SUPPORT
/**
* @brief Conversion from wide character string to a single-byte chain.
* @param wstr A wide-char string
* @return A multi-byte string
*/
INFERENCE_ENGINE_API_CPP(std::string) wStringtoMBCSstringChar(const std::wstring& wstr);
/**
* @brief Conversion from single-byte chain to wide character string.
* @param str A null-terminated string
* @return A wide-char string
*/
INFERENCE_ENGINE_API_CPP(std::wstring) multiByteCharToWString(const char* str);
#endif // ENABLE_UNICODE_PATH_SUPPORT
template <typename T> struct FileTraits;
#ifdef _WIN32
@ -83,7 +100,7 @@ INFERENCE_ENGINE_API(long long) fileSize(const char *fileName);
* @return { description_of_the_return_value }
*/
inline long long fileSize(const wchar_t* fileName) {
return fileSize(InferenceEngine::details::wStringtoMBCSstringChar(fileName).c_str());
return fileSize(::FileUtils::wStringtoMBCSstringChar(fileName).c_str());
}
#endif // ENABLE_UNICODE_PATH_SUPPORT
@ -167,11 +184,11 @@ inline std::basic_string<C> makeSharedLibraryName(const std::basic_string<C> &pa
using FilePath = std::wstring;
inline std::string fromFilePath(const FilePath & path) {
return InferenceEngine::details::wStringtoMBCSstringChar(path);
return ::FileUtils::wStringtoMBCSstringChar(path);
}
inline FilePath toFilePath(const std::string & path) {
return InferenceEngine::details::multiByteCharToWString(path.c_str());
return ::FileUtils::multiByteCharToWString(path.c_str());
}
#else

View File

@ -10,17 +10,18 @@
#pragma once
#include <cstdlib>
#include <details/os/os_filesystem.hpp>
#include <fstream>
#include <ie_precision.hpp>
#include <memory>
#include <pugixml.hpp>
#include <sstream>
#include <string>
#include <utility>
#include <pugixml.hpp>
#include "ie_api.h"
#include "ie_precision.hpp"
#include "ie_common.h"
#include "file_utils.h"
/**
* @ingroup ie_dev_api_xml
@ -253,8 +254,8 @@ struct parse_result {
* @return The parse_result.
*/
static parse_result ParseXml(const char* file_path) {
#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
std::wstring wFilePath = InferenceEngine::details::multiByteCharToWString(file_path);
#ifdef ENABLE_UNICODE_PATH_SUPPORT
std::wstring wFilePath = FileUtils::multiByteCharToWString(file_path);
const wchar_t* resolvedFilepath = wFilePath.c_str();
#else
const char* resolvedFilepath = file_path;

View File

@ -14,7 +14,6 @@
#include <utility>
#include <vector>
#include "details/os/os_filesystem.hpp"
#include "ie_format_parser.h"
#include "ie_ir_itt.hpp"
#include "parsers.h"
@ -73,7 +72,7 @@ void readAllFile(const std::string& string_file_name, void* buffer, size_t maxSi
std::ifstream inputFile;
#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
std::wstring file_name = InferenceEngine::details::multiByteCharToWString(string_file_name.c_str());
std::wstring file_name = FileUtils::multiByteCharToWString(string_file_name.c_str());
#else
std::string file_name = string_file_name;
#endif

View File

@ -123,12 +123,12 @@ TEST_P(NetReaderTest, ReadCorrectModelWithWeightsUnicodePath) {
is_copy_successfully = CommonTestUtils::copyFile(_modelPath, modelPath);
if (!is_copy_successfully) {
FAIL() << "Unable to copy from '" << _modelPath << "' to '"
<< InferenceEngine::details::wStringtoMBCSstringChar(modelPath) << "'";
<< FileUtils::wStringtoMBCSstringChar(modelPath) << "'";
}
is_copy_successfully = CommonTestUtils::copyFile(_weightsPath, weightsPath);
if (!is_copy_successfully) {
FAIL() << "Unable to copy from '" << _weightsPath << "' to '"
<< InferenceEngine::details::wStringtoMBCSstringChar(weightsPath) << "'";
<< FileUtils::wStringtoMBCSstringChar(weightsPath) << "'";
}
GTEST_COUT << "Test " << testIndex << std::endl;
InferenceEngine::Core ie;

View File

@ -276,14 +276,14 @@ TEST_P(IEClassBasicTestP, smoke_registerPluginsXMLUnicodePath) {
bool is_copy_successfully;
is_copy_successfully = CommonTestUtils::copyFile(pluginXML, pluginsXmlW);
if (!is_copy_successfully) {
FAIL() << "Unable to copy from '" << pluginXML << "' to '" << wStringtoMBCSstringChar(pluginsXmlW) << "'";
FAIL() << "Unable to copy from '" << pluginXML << "' to '" << ::FileUtils::wStringtoMBCSstringChar(pluginsXmlW) << "'";
}
GTEST_COUT << "Test " << testIndex << std::endl;
Core ie;
GTEST_COUT << "Core created " << testIndex << std::endl;
ASSERT_NO_THROW(ie.RegisterPlugins(wStringtoMBCSstringChar(pluginsXmlW)));
ASSERT_NO_THROW(ie.RegisterPlugins(::FileUtils::wStringtoMBCSstringChar(pluginsXmlW)));
CommonTestUtils::removeFile(pluginsXmlW);
#if defined __linux__ && !defined(__APPLE__)
ASSERT_NO_THROW(ie.GetVersions("mock")); // from pluginXML

View File

@ -9,7 +9,7 @@
#include <fstream>
#include <algorithm>
#include <details/os/os_filesystem.hpp>
#include <file_utils.h>
#ifdef ENABLE_UNICODE_PATH_SUPPORT
namespace CommonTestUtils {
@ -23,15 +23,13 @@ static void fixSlashes(std::wstring &str) {
}
static std::wstring stringToWString(std::string input) {
std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
std::wstring result = converter.from_bytes(input);
return result;
return ::FileUtils::multiByteCharToWString(input.c_str());
}
static bool copyFile(std::wstring source_path, std::wstring dest_path) {
#ifndef _WIN32
std::ifstream source(InferenceEngine::details::wStringtoMBCSstringChar(source_path), std::ios::binary);
std::ofstream dest(InferenceEngine::details::wStringtoMBCSstringChar(dest_path), std::ios::binary);
std::ifstream source(FileUtils::wStringtoMBCSstringChar(source_path), std::ios::binary);
std::ofstream dest(FileUtils::wStringtoMBCSstringChar(dest_path), std::ios::binary);
#else
fixSlashes(source_path);
fixSlashes(dest_path);
@ -68,7 +66,7 @@ static void removeFile(std::wstring path) {
#ifdef _WIN32
result = _wremove(path.c_str());
#else
result = remove(InferenceEngine::details::wStringtoMBCSstringChar(path).c_str());
result = remove(FileUtils::wStringtoMBCSstringChar(path).c_str());
#endif
}
}