IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use jobs pool for PDPD model conversion as well (#7602)

Browse Source 
* Revert "Combine all PDPD model generation scripts into one python command (#7349)"

This reverts commit 1eca8a6e49.

* Added jobs pool for PDPD model conversion

* Use jobs pool for PDPD model conversion as well
This commit is contained in:
Ilya Lavrenov 2021-09-23 10:40:55 +03:00 committed by GitHub
parent 359cff31aa
commit 1390440256
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
ngraph/test/frontend/paddlepaddle/CMakeLists.txt
View File

@ -29,17 +29,24 @@ target_compile_definitions(${TARGET_NAME} PRIVATE -D TEST_PADDLE_MODELS_DIRNAME=
# 'paddlepaddle' installed to check code style
if (paddlepaddle_FOUND)
set(TEST_PADDLE_MODELS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_PADDLE_MODELS_DIRNAME}/)
set(GEN_WRAPPER ${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_wrapper.py)
file(GLOB_RECURSE PADDLE_GEN_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_scripts/generate_*.py)
file(GLOB_RECURSE PADDLE_ALL_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/*.py)
set(OUT_FILE ${TEST_PADDLE_MODELS}/generate_done.txt)
add_custom_command(OUTPUT ${OUT_FILE}
COMMAND ${PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_wrapper.py
${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_scripts
${TEST_PADDLE_MODELS}
DEPENDS ${PADDLE_ALL_SCRIPTS}
)
add_custom_target(paddlepaddle_test_models DEPENDS ${OUT_FILE})
foreach(GEN_SCRIPT ${PADDLE_GEN_SCRIPTS})
get_filename_component(FILE_WE ${GEN_SCRIPT} NAME_WE)
set(OUT_DONE_FILE ${TEST_PADDLE_MODELS}/${FILE_WE}_done.txt)
list(APPEND OUT_FILES ${OUT_DONE_FILE})
add_custom_command(OUTPUT ${OUT_DONE_FILE}
COMMAND ${PYTHON_EXECUTABLE}
${GEN_WRAPPER}
${GEN_SCRIPT}
${TEST_PADDLE_MODELS}
${OUT_DONE_FILE}
DEPENDS ${GEN_WRAPPER} ${GEN_SCRIPT}
JOB_POOL four_jobs
COMMENT "Running ${FILE_WE}.py to generate PDPD model")
endforeach()
add_custom_target(paddlepaddle_test_models DEPENDS ${OUT_FILES})
install(DIRECTORY ${TEST_PADDLE_MODELS}
DESTINATION tests/${TEST_PADDLE_MODELS_DIRNAME}

24
ngraph/test/frontend/paddlepaddle/test_models/gen_wrapper.py
View File

@ -1,25 +1,19 @@
import glob
import os
import subprocess
import sys
print(sys.argv)
if len(sys.argv) < 3:
print("Gen folder and output folder must be specified as arguments")
sys.exit(1)
# print(sys.argv)
if len(sys.argv) < 4:
print("Script, output folder and mark file must be specified as arguments")
exit(1)
gen_folder = sys.argv[1]
gen_script = sys.argv[1]
out_folder = sys.argv[2]
mark_file = os.path.join(out_folder, "generate_done.txt")
mark_file = sys.argv[3]
gen_files = glob.glob(os.path.join(gen_folder, '**/generate_*.py'), recursive=True)
for gen_script in gen_files:
print("Processing: {} ".format(gen_script))
status = subprocess.run([sys.executable, gen_script, out_folder], env=os.environ)
if status.returncode != 0:
print("ERROR: PaddlePaddle model gen script FAILED: {}".format(gen_script))
sys.exit(1)
# print("Processing: {} ".format(gen_script))
subprocess.run([sys.executable, gen_script, out_folder], env=os.environ)
# Create mark file indicating that script was executed
with open(mark_file, "w") as fp: