IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] fixed loop serialization logic for multi-stream execution (#16838)

Browse Source 
* fixed loop serialization logic for multi-stream execution

* fixed the multistream unit test
This commit is contained in:
Eddy Kim
2023-04-11 12:40:37 +04:00
committed by GitHub
parent f991f92f8c
commit f6ee6e92f8
11 changed files with 158 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugins/intel_gpu/include/intel_gpu/graph/network.hpp
+2 -2
View File
@@ -87,8 +87,8 @@ public:
network(program::ptr program, stream::ptr stream, uint16_t stream_id);
network(cldnn::BinaryInputBuffer& ifs, stream::ptr stream, engine& engine, uint16_t stream_id = 0);
network(cldnn::BinaryInputBuffer& ifs, const ExecutionConfig& config, stream::ptr stream, engine& engine, uint16_t stream_id = 0);
network(cldnn::BinaryInputBuffer& ifs, stream::ptr stream, engine& engine, bool is_primary_stream = true);
network(cldnn::BinaryInputBuffer& ifs, const ExecutionConfig& config, stream::ptr stream, engine& engine, bool is_primary_stream = true);
~network();
src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp
+12 -4
View File
@@ -12,6 +12,8 @@
#include "bind.hpp"
namespace cldnn {
struct memory;
class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
public:
BinaryOutputBuffer(std::ostream& stream)
@@ -34,7 +36,7 @@ private:
class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
public:
BinaryInputBuffer(std::istream& stream, engine& engine)
: InputBuffer(this, engine), stream(stream), _impl_params(nullptr), _network(nullptr) {}
: InputBuffer(this, engine), stream(stream), _impl_params(nullptr) {}
void read(void* const data, std::streamsize size) {
auto const read_size = stream.rdbuf()->sgetn(reinterpret_cast<char*>(data), size);
@@ -44,8 +46,14 @@ public:
void setKernelImplParams(void* impl_params) { _impl_params = impl_params; }
void* getKernelImplParams() const { return _impl_params; }
void setNetwork(void* network) { _network = network; }
void* getNetwork() const { return _network; }
void addConstData(const std::string& prim_id, const std::shared_ptr<memory> mem_ptr) {
OPENVINO_ASSERT(_const_data_map.find(prim_id) == _const_data_map.end(), "[GPU] duplicated primitive id " + prim_id);
_const_data_map[prim_id] = mem_ptr;
}
std::shared_ptr<memory> getConstData(const std::string& prim_id) {
OPENVINO_ASSERT(_const_data_map.find(prim_id) != _const_data_map.end(), "[GPU] Not found primitive id " + prim_id);
return _const_data_map[prim_id];
}
std::streampos tellg() { return stream.tellg(); }
void seekg(std::streampos pos) { stream.seekg(pos); }
@@ -53,7 +61,7 @@ public:
private:
std::istream& stream;
void* _impl_params;
void* _network;
std::unordered_map<std::string, std::shared_ptr<memory>> _const_data_map;
};
template <typename T>
src/plugins/intel_gpu/include/intel_gpu/graph/serialization/utils.hpp
+9
View File
@@ -56,6 +56,15 @@ protected:
return (_pos < _buf.size()) ? _buf[_pos++] : EOF;
}
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which = std::ios_base::in) override {
return _pos;
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which = std::ios_base::in) override {
_pos = pos;
return _pos;
}
private:
std::vector<int_type> _buf;
size_t _pos;
src/plugins/intel_gpu/src/graph/data.cpp
+4 -3
View File
@@ -86,9 +86,8 @@ void data_inst::load(BinaryInputBuffer& ib) {
size_t data_size = 0;
ib >> make_data(&data_size, sizeof(size_t));
if (ib.getNetwork()) {
const network* primary_network = reinterpret_cast<network*>(ib.getNetwork());
_outputs[0] = primary_network->get_primitive(id())->output_memory_ptr();
if (!get_network().is_primary_stream()) {
_outputs[0] = ib.getConstData(id());
auto pos = ib.tellg();
pos += data_size;
ib.seekg(pos);
@@ -103,6 +102,8 @@ void data_inst::load(BinaryInputBuffer& ib) {
ib >> make_data(_buf.data(), data_size);
_outputs[0]->copy_from(get_network().get_stream(), _buf.data());
}
ib.addConstData(id(), _outputs[0]);
}
}
src/plugins/intel_gpu/src/graph/loop.cpp
+1 -1
View File
@@ -529,7 +529,7 @@ void loop_inst::load(BinaryInputBuffer& ib) {
ib >> _condition_id;
ib >> _num_iteration_id;
ib >> _max_iteration;
body_network = std::make_shared<cldnn::network>(ib, get_network().get_stream_ptr(), get_network().get_engine());
body_network = std::make_shared<cldnn::network>(ib, get_network().get_stream_ptr(), get_network().get_engine(), get_network().is_primary_stream());
}
} // namespace cldnn
src/plugins/intel_gpu/src/graph/mutable_data.cpp
+13 -11
View File
@@ -51,19 +51,21 @@ std::string mutable_data_inst::to_string(mutable_data_node const& node) {
}
void mutable_data_inst::set_output_memory(memory::ptr mem_new, bool check, size_t idx) {
auto& eng = _network.get_engine();
auto& mem_node = const_cast<program_node *>(_node)->as<mutable_data>();
auto& mem_attached = mem_node.get_attached_memory();
const auto& mem_orig = *_outputs[idx];
if (_node != nullptr) {
auto& eng = _network.get_engine();
auto& mem_node = const_cast<program_node *>(_node)->as<mutable_data>();
auto& mem_attached = mem_node.get_attached_memory();
const auto& mem_orig = *_outputs[idx];
if (!eng.is_the_same_buffer(*mem_new, mem_attached)) {
if (_node->is_input()) {
mem_new->copy_from(_network.get_stream(), *_outputs[idx]);
}
if (!eng.is_the_same_buffer(*mem_new, mem_attached)) {
if (_node->is_input()) {
mem_new->copy_from(_network.get_stream(), *_outputs[idx]);
}
// re-attach mutable_data internal memory if necessary
if (eng.is_the_same_buffer(mem_orig, mem_attached)) {
mem_node.attach_memory(eng.reinterpret_buffer(*mem_new, mem_attached.get_layout()));
// re-attach mutable_data internal memory if necessary
if (eng.is_the_same_buffer(mem_orig, mem_attached)) {
mem_node.attach_memory(eng.reinterpret_buffer(*mem_new, mem_attached.get_layout()));
}
}
}
primitive_inst::set_output_memory(mem_new, check);
src/plugins/intel_gpu/src/graph/network.cpp
+4 -4
View File
@@ -360,17 +360,17 @@ network::network(program::ptr program, uint16_t stream_id)
network::network(program::ptr program, stream::ptr stream, uint16_t stream_id)
: network(program, program->get_config(), stream, false, stream_id == 0) {}
network::network(cldnn::BinaryInputBuffer& ib, stream::ptr stream, engine& engine, uint16_t stream_id)
: network(ib, ExecutionConfig{}, stream, engine, stream_id) {}
network::network(cldnn::BinaryInputBuffer& ib, stream::ptr stream, engine& engine, bool is_primary_stream)
: network(ib, ExecutionConfig{}, stream, engine, is_primary_stream) {}
network::network(cldnn::BinaryInputBuffer& ib, const ExecutionConfig& config, stream::ptr stream, engine& engine, uint16_t stream_id)
network::network(cldnn::BinaryInputBuffer& ib, const ExecutionConfig& config, stream::ptr stream, engine& engine, bool is_primary_stream)
: _program(nullptr)
, _config(config)
, _engine(engine)
, _stream(stream)
, _memory_pool(new memory_pool(engine))
, _internal(false)
, _is_primary_stream(false)
, _is_primary_stream(is_primary_stream)
, _reset_arguments(true) {
net_id = get_unique_net_id();
src/plugins/intel_gpu/src/plugin/compiled_model.cpp
-3
View File
@@ -92,9 +92,6 @@ CompiledModel::CompiledModel(cldnn::BinaryInputBuffer& ib, InferenceEngine::Remo
ib.seekg(pos);
auto graph = std::make_shared<Graph>(ib, context_impl, m_config, n);
m_graphs.push_back(graph);
if (n == 0) {
ib.setNetwork(graph->GetNetwork().get());
}
}
}
src/plugins/intel_gpu/src/plugin/graph.cpp
+1 -1
View File
@@ -105,7 +105,7 @@ Graph::Graph(cldnn::BinaryInputBuffer &ib, RemoteContextImpl::Ptr context, const
size_t num_networks;
ib >> num_networks;
for (size_t i = 0; i < num_networks; ++i) {
m_networks.emplace_back(std::make_shared<cldnn::network>(ib, get_engine().create_stream(config), get_engine(), m_stream_id));
m_networks.emplace_back(std::make_shared<cldnn::network>(ib, get_engine().create_stream(config), get_engine(), m_stream_id == 0));
}
}
src/plugins/intel_gpu/tests/test_cases/streams_test.cpp
+10 -19
View File
@@ -79,33 +79,24 @@ public:
cldnn::network::ptr network1;
if (is_caching_test) {
std::cout << "cached" << std::endl;
membuf mem_buf0;
membuf mem_buf1;
membuf mem_buf;
{
auto prog = program::build_program(engine, topology, get_test_default_config(engine));
{
network0 = std::make_shared<cldnn::network>(prog, 0);
std::ostream out_mem0(&mem_buf0);
BinaryOutputBuffer ob0 = BinaryOutputBuffer(out_mem0);
network0->save(ob0);
}
{
network1 = std::make_shared<cldnn::network>(prog, 1);
std::ostream out_mem1(&mem_buf1);
BinaryOutputBuffer ob1 = BinaryOutputBuffer(out_mem1);
network1->save(ob1);
std::ostream out_mem(&mem_buf);
BinaryOutputBuffer ob = BinaryOutputBuffer(out_mem);
network0->save(ob);
}
}
{
{
std::istream in_mem0(&mem_buf0);
BinaryInputBuffer ib0 = BinaryInputBuffer(in_mem0, engine);
network0 = std::make_shared<cldnn::network>(ib0, get_test_stream_ptr(), engine, 0);
}
{
std::istream in_mem1(&mem_buf1);
BinaryInputBuffer ib1 = BinaryInputBuffer(in_mem1, engine);
network1 = std::make_shared<cldnn::network>(ib1, get_test_stream_ptr(), engine, 1);
std::istream in_mem(&mem_buf);
BinaryInputBuffer ib = BinaryInputBuffer(in_mem, engine);
auto pos = ib.tellg();
network0 = std::make_shared<cldnn::network>(ib, get_test_stream_ptr(), engine, true);
ib.seekg(pos);
network1 = std::make_shared<cldnn::network>(ib, get_test_stream_ptr(), engine, false);
}
}
} else {
src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp
+102 -55
View File
@@ -16,6 +16,8 @@
#include "functional_test_utils/blob_utils.hpp"
#include "openvino/core/preprocess/pre_post_process.hpp"
#include "transformations/utils/utils.hpp"
#include "common_test_utils/file_utils.hpp"
#include "openvino/runtime/intel_gpu/properties.hpp"
using namespace ::testing;
@@ -27,7 +29,8 @@ class OVConcurrencyTest : public CommonTestUtils::TestsCommon,
void SetUp() override {
std::tie(num_streams, num_requests) = this->GetParam();
fn_ptrs = {ngraph::builder::subgraph::makeSplitMultiConvConcat(),
ngraph::builder::subgraph::makeMultiSingleConv()};
ngraph::builder::subgraph::makeMultiSingleConv(),
ngraph::builder::subgraph::makeTIwithLSTMcell()};
};
public:
static std::string getTestCaseName(const testing::TestParamInfo<ConcurrencyTestParams>& obj) {
@@ -37,6 +40,100 @@ public:
std::to_string(requests);
}
void execute(bool is_caching_test = false) {
auto ie = ov::Core();
std::string cacheFolderName;
if (is_caching_test) {
std::stringstream ss;
ss << "OVConcurrencyTest_nstreams_" << num_streams << "_nireq_" << num_requests;
cacheFolderName = ss.str();
CommonTestUtils::removeFilesWithExt(cacheFolderName, "blob");
CommonTestUtils::removeFilesWithExt(cacheFolderName, "cl_cache");
CommonTestUtils::removeDir(cacheFolderName);
ie.set_property(ov::cache_dir(cacheFolderName));
ie.set_property(ov::intel_gpu::enable_loop_unrolling(false));
}
ov::ResultVector outputs;
std::vector<ov::InferRequest> irs;
std::vector<std::vector<uint8_t>> ref;
std::vector<int> outElementsCount;
for (size_t i = 0; i < fn_ptrs.size(); ++i) {
auto fn = fn_ptrs[i];
ov::CompiledModel exec_net;
if (is_caching_test) {
{
auto _dummy_exec_net = ie.compile_model(fn_ptrs[i], CommonTestUtils::DEVICE_GPU,
ov::num_streams(num_streams), ov::hint::inference_precision(ov::element::f32));
}
{
exec_net = ie.compile_model(fn_ptrs[i], CommonTestUtils::DEVICE_GPU,
ov::num_streams(num_streams), ov::hint::inference_precision(ov::element::f32));
}
} else {
exec_net = ie.compile_model(fn_ptrs[i], CommonTestUtils::DEVICE_GPU,
ov::num_streams(num_streams), ov::hint::inference_precision(ov::element::f32));
}
auto output = fn_ptrs[i]->get_results().at(0);
for (int j = 0; j < num_streams * num_requests; j++) {
outputs.push_back(output);
auto inf_req = exec_net.create_infer_request();
irs.push_back(inf_req);
std::vector<std::vector<uint8_t>> inputs;
for (size_t param_idx = 0; param_idx < fn_ptrs[i]->get_parameters().size(); ++param_idx) {
auto input = fn_ptrs[i]->get_parameters().at(param_idx);
auto tensor = FuncTestUtils::create_and_fill_tensor(input->get_element_type(), input->get_shape());
inf_req.set_tensor(input, tensor);
const auto in_tensor = inf_req.get_tensor(input);
const auto tensorSize = in_tensor.get_byte_size();
const auto inBlobBuf = static_cast<uint8_t*>(in_tensor.data());
std::vector<uint8_t> inData(inBlobBuf, inBlobBuf + tensorSize);
inputs.emplace_back(inData);
}
auto reOutData = ngraph::helpers::interpreterFunction(fn_ptrs[i], inputs).front().second;
ref.push_back(reOutData);
outElementsCount.push_back(ov::shape_size(fn_ptrs[i]->get_output_shape(0)));
}
}
const int niter = 10;
for (int i = 0; i < niter; i++) {
for (auto ir : irs) {
ir.start_async();
}
for (auto ir : irs) {
ir.wait();
}
}
auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
for (size_t i = 0; i < irs.size(); ++i) {
const auto &refBuffer = ref[i].data();
ASSERT_EQ(outElementsCount[i], irs[i].get_tensor(outputs[i]).get_size());
FuncTestUtils::compareRawBuffers(irs[i].get_tensor(outputs[i]).data<float>(),
reinterpret_cast<const float *>(refBuffer), outElementsCount[i],
outElementsCount[i],
thr);
}
if (is_caching_test) {
CommonTestUtils::removeFilesWithExt(cacheFolderName, "blob");
CommonTestUtils::removeFilesWithExt(cacheFolderName, "cl_cache");
CommonTestUtils::removeDir(cacheFolderName);
}
}
protected:
size_t num_streams;
size_t num_requests;
@@ -44,61 +141,11 @@ protected:
};
TEST_P(OVConcurrencyTest, canInferTwoExecNets) {
auto ie = ov::Core();
this->execute(false);
}
ov::ResultVector outputs;
std::vector<ov::InferRequest> irs;
std::vector<std::vector<uint8_t>> ref;
std::vector<int> outElementsCount;
for (size_t i = 0; i < fn_ptrs.size(); ++i) {
auto fn = fn_ptrs[i];
auto exec_net = ie.compile_model(fn_ptrs[i], CommonTestUtils::DEVICE_GPU,
ov::num_streams(num_streams), ov::hint::inference_precision(ov::element::f32));
auto input = fn_ptrs[i]->get_parameters().at(0);
auto output = fn_ptrs[i]->get_results().at(0);
for (int j = 0; j < num_streams * num_requests; j++) {
outputs.push_back(output);
auto inf_req = exec_net.create_infer_request();
irs.push_back(inf_req);
auto tensor = FuncTestUtils::create_and_fill_tensor(input->get_element_type(), input->get_shape());
inf_req.set_tensor(input, tensor);
outElementsCount.push_back(ov::shape_size(fn_ptrs[i]->get_output_shape(0)));
const auto in_tensor = inf_req.get_tensor(input);
const auto tensorSize = in_tensor.get_byte_size();
const auto inBlobBuf = static_cast<uint8_t*>(in_tensor.data());
std::vector<uint8_t> inData(inBlobBuf, inBlobBuf + tensorSize);
auto reOutData = ngraph::helpers::interpreterFunction(fn_ptrs[i], {inData}).front().second;
ref.push_back(reOutData);
}
}
const int niter = 10;
for (int i = 0; i < niter; i++) {
for (auto ir : irs) {
ir.start_async();
}
for (auto ir : irs) {
ir.wait();
}
}
auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
for (size_t i = 0; i < irs.size(); ++i) {
const auto &refBuffer = ref[i].data();
ASSERT_EQ(outElementsCount[i], irs[i].get_tensor(outputs[i]).get_size());
FuncTestUtils::compareRawBuffers(irs[i].get_tensor(outputs[i]).data<float>(),
reinterpret_cast<const float *>(refBuffer), outElementsCount[i],
outElementsCount[i],
thr);
}
TEST_P(OVConcurrencyTest, canInferTwoExecNets_cached) {
this->execute(true);
}
const std::vector<size_t> num_streams{ 1, 2 };