From 82cdc116c1c9ba161de0b70ca4baadc72ef96a4b Mon Sep 17 00:00:00 2001 From: Sebastian Golebiewski Date: Wed, 13 Dec 2023 10:33:50 +0100 Subject: [PATCH] Updating Interactive Tutorials (#21548) --- docs/articles_en/learn_openvino/tutorials.rst | 85 +- .../tutorials/notebooks_section_1.rst | 6 + .../tutorials/notebooks_section_2.rst | 28 + docs/nbdoc/consts.py | 2 +- .../002-openvino-api-with-output.rst | 184 +- ...2-pytorch-onnx-to-openvino-with-output.rst | 115 +- .../102-pytorch-to-openvino-with-output.rst | 796 -------- .../index.html | 32 +- ...to-openvino-classification-with-output.rst | 115 +- ...envino-classification-with-output_15_3.png | 3 + ...envino-classification-with-output_16_3.png | 3 - ...envino-classification-with-output_23_1.png | 3 + ...envino-classification-with-output_24_1.png | 3 - ...envino-classification-with-output_27_1.png | 3 + ...envino-classification-with-output_28_1.png | 3 - ...envino-classification-with-output_30_1.png | 3 + ...envino-classification-with-output_31_1.png | 3 - ...penvino-classification-with-output_8_1.png | 3 + ...penvino-classification-with-output_9_1.png | 3 - .../index.html | 14 +- ...105-language-quantize-bert-with-output.rst | 207 +- ...tion-quantization-data2vec-with-output.rst | 65 +- .../109-latency-tricks-with-output.rst | 153 +- ...> 109-latency-tricks-with-output_15_0.jpg} | 0 ...> 109-latency-tricks-with-output_18_0.jpg} | 0 ...> 109-latency-tricks-with-output_20_0.jpg} | 0 ...> 109-latency-tricks-with-output_24_0.jpg} | 0 ...> 109-latency-tricks-with-output_26_0.jpg} | 0 ...> 109-latency-tricks-with-output_28_0.jpg} | 0 .../109-latency-tricks-with-output_30_0.png | 3 - .../109-latency-tricks-with-output_31_0.png | 3 + ...=> 109-latency-tricks-with-output_5_0.jpg} | 0 .../index.html | 20 +- .../109-throughput-tricks-with-output.rst | 157 +- ...09-throughput-tricks-with-output_15_0.jpg} | 0 ...09-throughput-tricks-with-output_18_0.jpg} | 0 ...09-throughput-tricks-with-output_21_0.jpg} | 0 ...09-throughput-tricks-with-output_25_0.jpg} | 0 ...09-throughput-tricks-with-output_29_0.jpg} | 0 ...09-throughput-tricks-with-output_31_0.jpg} | 0 ...109-throughput-tricks-with-output_33_0.png | 3 - ...109-throughput-tricks-with-output_34_0.png | 3 + ...109-throughput-tricks-with-output_5_0.jpg} | 0 .../index.html | 20 +- ...segmentation-quantize-nncf-with-output.rst | 249 ++- ...ntation-quantize-nncf-with-output_15_1.png | 2 +- ...ntation-quantize-nncf-with-output_37_1.png | 4 +- .../index.html | 10 +- ...training-quantization-nncf-with-output.rst | 245 ++- ...lassification-quantization-with-output.rst | 174 +- ...fication-quantization-with-output_30_2.png | 2 +- .../index.html | 6 +- docs/notebooks/115-async-api-with-output.rst | 21 +- .../115-async-api-with-output_21_0.png | 4 +- .../index.html | 12 +- .../116-sparsity-optimization-with-output.rst | 100 +- .../119-tflite-to-openvino-with-output.rst | 85 +- .../index.html | 8 +- ...e-segmentation-to-openvino-with-output.rst | 122 +- ...mentation-to-openvino-with-output_25_1.png | 2 +- ...mentation-to-openvino-with-output_39_0.png | 4 +- .../index.html | 8 +- ...ject-detection-to-openvino-with-output.rst | 114 +- ...detection-to-openvino-with-output_25_1.png | 2 +- ...detection-to-openvino-with-output_38_0.png | 4 +- .../index.html | 8 +- .../121-convert-to-openvino-with-output.rst | 101 +- ...tion-with-accuracy-control-with-output.rst | 107 +- ...123-detectron2-to-openvino-with-output.rst | 100 +- ...etectron2-to-openvino-with-output_22_0.jpg | 4 +- ...etectron2-to-openvino-with-output_22_0.png | 4 +- ...etectron2-to-openvino-with-output_32_0.jpg | 4 +- ...etectron2-to-openvino-with-output_32_0.png | 4 +- .../index.html | 16 +- .../124-hugging-face-hub-with-output.rst | 72 +- .../125-lraspp-segmentation-with-output.rst | 326 ++++ ...5-lraspp-segmentation-with-output_28_0.png | 3 + ...5-lraspp-segmentation-with-output_32_0.png | 3 + ...5-lraspp-segmentation-with-output_34_0.png | 3 + ...25-lraspp-segmentation-with-output_5_0.jpg | 3 + ...25-lraspp-segmentation-with-output_5_0.png | 3 + .../index.html | 11 + .../126-tensorflow-hub-with-output.rst | 25 +- .../126-tensorflow-hub-with-output_26_0.png | 2 +- .../126-tensorflow-hub-with-output_45_0.png | 2 +- .../index.html | 12 +- .../201-vision-monodepth-with-output.rst | 118 +- .../201-vision-monodepth-with-output_18_0.png | 2 +- .../index.html | 6 +- ...nter-semantic-segmentation-with-output.rst | 140 +- .../index.html | 12 +- ...slowfast-video-recognition-with-output.rst | 49 +- .../216-attention-center-with-output.rst | 347 ++++ .../216-attention-center-with-output_15_1.png | 3 + .../216-attention-center-with-output_17_1.png | 3 + .../index.html | 8 + ...ss-lingual-books-alignment-with-output.rst | 131 +- ...ngual-books-alignment-with-output_31_0.png | 2 +- ...ngual-books-alignment-with-output_48_0.png | 4 +- .../index.html | 8 +- ...-segmentation-point-clouds-with-output.rst | 58 +- ...entation-point-clouds-with-output_11_1.png | 3 - ...entation-point-clouds-with-output_11_2.png | 3 + ...entation-point-clouds-with-output_16_0.png | 3 - ...entation-point-clouds-with-output_16_1.png | 3 + .../index.html | 8 +- .../226-yolov7-optimization-with-output.rst | 243 ++- ...6-yolov7-optimization-with-output_10_0.jpg | 4 +- ...6-yolov7-optimization-with-output_10_0.png | 4 +- ...6-yolov7-optimization-with-output_27_0.jpg | 4 +- ...6-yolov7-optimization-with-output_27_0.png | 4 +- ...6-yolov7-optimization-with-output_44_0.jpg | 4 +- ...6-yolov7-optimization-with-output_44_0.png | 4 +- .../index.html | 16 +- .../227-whisper-convert-with-output.rst | 7 +- ...rt-sequence-classification-with-output.rst | 96 +- ...lov8-instance-segmentation-with-output.rst | 157 +- .../index.html | 18 +- ...-yolov8-keypoint-detection-with-output.rst | 157 +- .../index.html | 18 +- ...30-yolov8-object-detection-with-output.rst | 204 +- ...lov8-object-detection-with-output_22_0.jpg | 3 - .../index.html | 22 +- ...ruct-pix2pix-image-editing-with-output.rst | 88 +- .../index.html | 8 +- .../233-blip-convert-with-output.rst | 109 +- .../index.html | 10 +- ...-encodec-audio-compression-with-output.rst | 117 +- ...dec-audio-compression-with-output_19_1.png | 4 +- ...dec-audio-compression-with-output_38_1.png | 4 +- ...odec-audio-compression-with-output_6_2.png | 2 +- .../index.html | 10 +- ...ontrolnet-stable-diffusion-with-output.rst | 3 +- .../index.html | 12 +- ...diffusion-v2-infinite-zoom-with-output.rst | 80 +- ...sion-v2-text-to-image-demo-with-output.rst | 58 +- ...2-text-to-image-demo-with-output_24_0.jpg} | 0 ...2-text-to-image-demo-with-output_24_0.png} | 0 .../index.html | 8 +- ...diffusion-v2-text-to-image-with-output.rst | 76 +- .../237-segment-anything-with-output.rst | 141 +- .../index.html | 28 +- .../239-image-bind-convert-with-output.rst | 67 +- .../index.html | 22 +- ...ly-2-instruction-following-with-output.rst | 34 +- .../245-typo-detector-with-output.rst | 127 +- .../247-code-language-id-with-output.rst | 44 +- docs/notebooks/248-ssd-b1-with-output.rst | 336 ++++ .../248-ssd-b1-with-output_11_1.jpg | 3 + .../248-ssd-b1-with-output_11_1.png | 3 + .../248-ssd-b1-with-output_9_3.jpg | 3 + .../248-ssd-b1-with-output_9_3.png | 3 + .../248-ssd-b1-with-output_files/index.html | 10 + .../248-stable-diffusion-xl-with-output.rst | 81 +- .../index.html | 16 +- .../250-music-generation-with-output.rst | 24 +- ...1-tiny-sd-image-generation-with-output.rst | 78 +- .../index.html | 16 +- .../notebooks/254-llm-chatbot-with-output.rst | 750 ++++---- .../256-bark-text-to-audio-with-output.rst | 64 +- ...7-llava-multimodal-chatbot-with-output.rst | 13 +- .../index.html | 6 +- .../260-pix2struct-docvqa-with-output.rst | 5 +- .../index.html | 8 +- .../261-fast-segment-anything-with-output.rst | 77 +- .../index.html | 12 +- ...62-softvc-voice-conversion-with-output.rst | 6 +- ...cy-models-image-generation-with-output.rst | 214 ++- ...dels-image-generation-with-output_21_0.jpg | 3 + ...dels-image-generation-with-output_21_0.png | 4 +- ...dels-image-generation-with-output_34_1.jpg | 3 + ...dels-image-generation-with-output_34_1.png | 4 +- .../index.html | 10 +- .../263-lcm-lora-controlnet-with-output.rst | 1664 +++++++++++++++++ ...3-lcm-lora-controlnet-with-output_10_1.png | 3 + ...3-lcm-lora-controlnet-with-output_27_2.jpg | 3 + ...3-lcm-lora-controlnet-with-output_27_2.png | 3 + ...3-lcm-lora-controlnet-with-output_28_0.png | 3 + ...3-lcm-lora-controlnet-with-output_42_1.png | 3 + .../index.html | 11 + ...uerstchen-image-generation-with-output.rst | 5 +- .../index.html | 8 +- .../266-speculative-sampling-with-output.rst | 3 +- .../267-distil-whisper-asr-with-output.rst | 49 +- ...8-table-question-answering-with-output.rst | 77 +- .../notebooks/269-film-slowmo-with-output.rst | 45 +- .../index.html | 10 +- ...sound-generation-audioldm2-with-output.rst | 774 ++++++++ .../270-sound-generation-audioldm2.png | 3 + docs/notebooks/271-sdxl-turbo-with-output.rst | 860 +++++++++ .../271-sdxl-turbo-with-output_11_3.jpg | 3 + .../271-sdxl-turbo-with-output_11_3.png | 3 + .../271-sdxl-turbo-with-output_17_2.jpg | 3 + .../271-sdxl-turbo-with-output_17_2.png | 3 + .../271-sdxl-turbo-with-output_29_3.jpg | 3 + .../271-sdxl-turbo-with-output_29_3.png | 3 + .../271-sdxl-turbo-with-output_30_1.jpg | 3 + .../271-sdxl-turbo-with-output_30_1.png | 3 + .../index.html | 14 + .../272-paint-by-example-with-output.rst | 1104 +++++++++++ .../272-paint-by-example.png | 3 + ...uantization-aware-training-with-output.rst | 157 +- .../406-3D-pose-estimation-with-output.rst | 202 +- docs/notebooks/all_notebooks_paths.txt | 5 + docs/notebooks/index.html | 428 +++-- docs/notebooks/notebook_utils-with-output.rst | 39 +- .../index.html | 18 +- .../notebook_utils-with-output_26_0.png | 4 +- .../notebook_utils-with-output_41_0.png | 2 +- .../notebook_utils-with-output_41_1.png | 2 +- .../notebook_utils-with-output_41_2.png | 2 +- .../notebook_utils-with-output_41_3.png | 2 +- docs/notebooks/notebooks_tags.json | 24 +- 213 files changed, 10196 insertions(+), 4070 deletions(-) delete mode 100644 docs/notebooks/102-pytorch-to-openvino-with-output.rst create mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_15_3.png delete mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_16_3.png create mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png delete mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_24_1.png create mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png delete mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_28_1.png create mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png delete mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_31_1.png create mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png delete mode 100644 docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_9_1.png rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_14_0.jpg => 109-latency-tricks-with-output_15_0.jpg} (100%) rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_17_0.jpg => 109-latency-tricks-with-output_18_0.jpg} (100%) rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_19_0.jpg => 109-latency-tricks-with-output_20_0.jpg} (100%) rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_23_0.jpg => 109-latency-tricks-with-output_24_0.jpg} (100%) rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_25_0.jpg => 109-latency-tricks-with-output_26_0.jpg} (100%) rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_27_0.jpg => 109-latency-tricks-with-output_28_0.jpg} (100%) delete mode 100644 docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png create mode 100644 docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_31_0.png rename docs/notebooks/109-latency-tricks-with-output_files/{109-latency-tricks-with-output_4_0.jpg => 109-latency-tricks-with-output_5_0.jpg} (100%) rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_14_0.jpg => 109-throughput-tricks-with-output_15_0.jpg} (100%) rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_17_0.jpg => 109-throughput-tricks-with-output_18_0.jpg} (100%) rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_20_0.jpg => 109-throughput-tricks-with-output_21_0.jpg} (100%) rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_24_0.jpg => 109-throughput-tricks-with-output_25_0.jpg} (100%) rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_28_0.jpg => 109-throughput-tricks-with-output_29_0.jpg} (100%) rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_30_0.jpg => 109-throughput-tricks-with-output_31_0.jpg} (100%) delete mode 100644 docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_33_0.png create mode 100644 docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_34_0.png rename docs/notebooks/109-throughput-tricks-with-output_files/{109-throughput-tricks-with-output_4_0.jpg => 109-throughput-tricks-with-output_5_0.jpg} (100%) create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output.rst create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_28_0.png create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_32_0.png create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_34_0.png create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.jpg create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.png create mode 100644 docs/notebooks/125-lraspp-segmentation-with-output_files/index.html create mode 100644 docs/notebooks/216-attention-center-with-output.rst create mode 100644 docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_15_1.png create mode 100644 docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_17_1.png create mode 100644 docs/notebooks/216-attention-center-with-output_files/index.html delete mode 100644 docs/notebooks/224-3D-segmentation-point-clouds-with-output_files/224-3D-segmentation-point-clouds-with-output_11_1.png create mode 100644 docs/notebooks/224-3D-segmentation-point-clouds-with-output_files/224-3D-segmentation-point-clouds-with-output_11_2.png delete mode 100644 docs/notebooks/224-3D-segmentation-point-clouds-with-output_files/224-3D-segmentation-point-clouds-with-output_16_0.png create mode 100644 docs/notebooks/224-3D-segmentation-point-clouds-with-output_files/224-3D-segmentation-point-clouds-with-output_16_1.png delete mode 100644 docs/notebooks/230-yolov8-object-detection-with-output_files/230-yolov8-object-detection-with-output_22_0.jpg rename docs/notebooks/236-stable-diffusion-v2-text-to-image-demo-with-output_files/{236-stable-diffusion-v2-text-to-image-demo-with-output_25_0.jpg => 236-stable-diffusion-v2-text-to-image-demo-with-output_24_0.jpg} (100%) rename docs/notebooks/236-stable-diffusion-v2-text-to-image-demo-with-output_files/{236-stable-diffusion-v2-text-to-image-demo-with-output_25_0.png => 236-stable-diffusion-v2-text-to-image-demo-with-output_24_0.png} (100%) create mode 100644 docs/notebooks/248-ssd-b1-with-output.rst create mode 100644 docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.jpg create mode 100644 docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.png create mode 100644 docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.jpg create mode 100644 docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.png create mode 100644 docs/notebooks/248-ssd-b1-with-output_files/index.html create mode 100644 docs/notebooks/263-latent-consistency-models-image-generation-with-output_files/263-latent-consistency-models-image-generation-with-output_21_0.jpg create mode 100644 docs/notebooks/263-latent-consistency-models-image-generation-with-output_files/263-latent-consistency-models-image-generation-with-output_34_1.jpg create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output.rst create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output_files/263-lcm-lora-controlnet-with-output_10_1.png create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output_files/263-lcm-lora-controlnet-with-output_27_2.jpg create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output_files/263-lcm-lora-controlnet-with-output_27_2.png create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output_files/263-lcm-lora-controlnet-with-output_28_0.png create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output_files/263-lcm-lora-controlnet-with-output_42_1.png create mode 100644 docs/notebooks/263-lcm-lora-controlnet-with-output_files/index.html create mode 100644 docs/notebooks/270-sound-generation-audioldm2-with-output.rst create mode 100644 docs/notebooks/270-sound-generation-audioldm2-with-output_files/270-sound-generation-audioldm2.png create mode 100644 docs/notebooks/271-sdxl-turbo-with-output.rst create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_11_3.jpg create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_11_3.png create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_17_2.jpg create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_17_2.png create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_29_3.jpg create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_29_3.png create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_30_1.jpg create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/271-sdxl-turbo-with-output_30_1.png create mode 100644 docs/notebooks/271-sdxl-turbo-with-output_files/index.html create mode 100644 docs/notebooks/272-paint-by-example-with-output.rst create mode 100644 docs/notebooks/272-paint-by-example-with-output_files/272-paint-by-example.png diff --git a/docs/articles_en/learn_openvino/tutorials.rst b/docs/articles_en/learn_openvino/tutorials.rst index f2d86fdcec4..e5d8772f85a 100644 --- a/docs/articles_en/learn_openvino/tutorials.rst +++ b/docs/articles_en/learn_openvino/tutorials.rst @@ -58,30 +58,37 @@ The Jupyter notebooks are categorized into following classes: Below you will find a selection of recommended tutorials that demonstrate inference on a particular model. These tutorials are guaranteed to provide a great experience with inference in OpenVINO: + +.. showcase:: + :title: 272-paint-by-example + :img: https://camo.githubusercontent.com/79d83ce8fc6813a503f372bacb7dc709c79d9560060df3dc92196b9849cc33a9/68747470733a2f2f757365722d696d616765732e67697468756275736572636f6e74656e742e636f6d2f3130333232363538302f3233363935343931382d66333634623232372d323933632d346637382d613962662d3964636562636231303334612e706e67 + + Paint by Example using Stable Diffusion and OpenVINO. + +.. showcase:: + :title: 271-sdxl-turbo + :img: _images/271-sdxl-turbo-with-output_30_1.png + + Single step image generation using SDXL-turbo and OpenVINO. + +.. showcase:: + :title: 270-sound-generation-audioldm2 + :img: https://github.com/openvinotoolkit/openvino_notebooks/assets/76463150/c93a0f86-d9cf-4bd1-93b9-e27532170d75 + + Sound Generation with AudioLDM2 and OpenVINO. + .. showcase:: :title: 269-film-slowmo :img: https://github.com/googlestaging/frame-interpolation/raw/main/moment.gif Frame interpolation using FILM and OpenVINO. -.. showcase:: - :title: 268-table-question-answering - :img: _static/images/notebook_eye.png - - Table Question Answering using TAPAS and OpenVINO. - .. showcase:: :title: 267-distil-whisper-asr :img: _static/images/notebook_eye.png Automatic speech recognition using Distil-Whisper and OpenVINO. -.. showcase:: - :title: 266-speculative-sampling - :img: _static/images/notebook_eye.png - - Text Generation via Speculative Sampling, KV Caching, and OpenVINO. - .. showcase:: :title: 265-wuerstchen-image-generation :img: https://user-images.githubusercontent.com/76161256/277724498-6917c558-d74c-4cc9-b81a-679ce0a299ee.png @@ -100,60 +107,18 @@ Below you will find a selection of recommended tutorials that demonstrate infere Image generation with Latent Consistency Model and OpenVINO. +.. showcase:: + :title: 263-lcm-lora-controlnet + :img: https://user-images.githubusercontent.com/29454499/284292122-f146e16d-7233-49f7-a401-edcb714b5288.png + + Text-to-Image Generation with LCM LoRA and ControlNet Conditioning. + .. showcase:: :title: 262-softvc-voice-conversion :img: _static/images/notebook_eye.png SoftVC VITS Singing Voice Conversion and OpenVINO. -.. showcase:: - :title: 261-fast-segment-anything - :img: https://user-images.githubusercontent.com/26833433/248551984-d98f0f6d-7535-45d0-b380-2e1440b52ad7.jpg - - Object segmentation with FastSAM and OpenVINO. - -.. showcase:: - :title: 259-decidiffusion-image-generation - :img: https://user-images.githubusercontent.com/29454499/274927904-cd734349-9954-4656-ab96-08a903e846ef.png - - Image generation with DeciDiffusion and OpenVINO. - -.. showcase:: - :title: 258-blip-diffusion-subject-generation - :img: https://user-images.githubusercontent.com/76161256/275485611-0ecf621f-b544-44ae-8258-8a49be704989.png - - Subject-driven image generation and editing using BLIP Diffusion and OpenVINO. - -.. showcase:: - :title: 257-llava-multimodal-chatbot - :img: https://raw.githubusercontent.com/haotian-liu/LLaVA/main/images/llava_logo.png - - Visual-language assistant with LLaVA and OpenVINO. - -.. showcase:: - :title: 256-bark-text-to-audio - :img: https://user-images.githubusercontent.com/29454499/269278630-9a770279-0045-480e-95f2-1a2f2d0a5115.png - - Text-to-speech generation using Bark and OpenVINO. - -.. showcase:: - :title: 254-llm-chatbot - :img: _static/images/notebook_eye.png - - Create an LLM-powered Chatbot using OpenVINO. - -.. showcase:: - :title: 253-zeroscope-text2video - :img: https://camo.githubusercontent.com/64eec6e52d060ca971c5a3be3f0d60e712907c98b4661b454d7e3e9575c2bc6b/68747470733a2f2f68756767696e67666163652e636f2f64617461736574732f68756767696e67666163652f646f63756d656e746174696f6e2d696d616765732f7265736f6c76652f6d61696e2f6469666675736572732f646172746876616465725f63657270656e73652e676966 - - Video generation with ZeroScope and OpenVINO. - -.. showcase:: - :title: 251-tiny-sd-image-generation - :img: https://user-images.githubusercontent.com/29454499/260904650-274fc2f9-24d2-46a3-ac3d-d660ec3c9a19.png - - Image Generation with Tiny-SD and OpenVINO. - .. note:: If there are any issues while running the notebooks, refer to the **Troubleshooting** and **FAQ** sections in the :doc:`Installation Guide ` or start a GitHub diff --git a/docs/articles_en/learn_openvino/tutorials/notebooks_section_1.rst b/docs/articles_en/learn_openvino/tutorials/notebooks_section_1.rst index 7d1bb5a25d3..54a4b4a35e3 100644 --- a/docs/articles_en/learn_openvino/tutorials/notebooks_section_1.rst +++ b/docs/articles_en/learn_openvino/tutorials/notebooks_section_1.rst @@ -17,6 +17,12 @@ Tutorials that explain how to optimize and quantize models with OpenVINO tools. Convert TensorFlow Hub models to OpenVINO Intermediate Representation (IR). +.. showcase:: + :title: 125-lraspp-segmentation + :img: _static/images/notebook_eye.png + + Semantic segmentation with LRASPP MobileNet v3 and OpenVINO + .. showcase:: :title: 125-convnext-classification :img: _static/images/notebook_eye.png diff --git a/docs/articles_en/learn_openvino/tutorials/notebooks_section_2.rst b/docs/articles_en/learn_openvino/tutorials/notebooks_section_2.rst index 18ab9430db1..b3c03f09a55 100644 --- a/docs/articles_en/learn_openvino/tutorials/notebooks_section_2.rst +++ b/docs/articles_en/learn_openvino/tutorials/notebooks_section_2.rst @@ -11,6 +11,22 @@ Model Demos Demos that demonstrate inference on a particular model. +.. showcase:: + :title: 272-paint-by-example + :img: https://camo.githubusercontent.com/79d83ce8fc6813a503f372bacb7dc709c79d9560060df3dc92196b9849cc33a9/68747470733a2f2f757365722d696d616765732e67697468756275736572636f6e74656e742e636f6d2f3130333232363538302f3233363935343931382d66333634623232372d323933632d346637382d613962662d3964636562636231303334612e706e67 + + Paint by Example using Stable Diffusion and OpenVINO. + +.. showcase:: + :title: 271-sdxl-turbo + :img: _images/271-sdxl-turbo-with-output_30_1.png + + Single step image generation using SDXL-turbo and OpenVINO. + +.. showcase:: + :title: 270-sound-generation-audioldm2 + :img: https://github.com/openvinotoolkit/openvino_notebooks/assets/76463150/c93a0f86-d9cf-4bd1-93b9-e27532170d75 + .. showcase:: :title: 269-film-slowmo :img: https://github.com/googlestaging/frame-interpolation/raw/main/moment.gif @@ -53,6 +69,12 @@ Demos that demonstrate inference on a particular model. Image generation with Latent Consistency Model and OpenVINO. +.. showcase:: + :title: 263-lcm-lora-controlnet + :img: https://user-images.githubusercontent.com/29454499/284292122-f146e16d-7233-49f7-a401-edcb714b5288.png + + Text-to-Image Generation with LCM LoRA and ControlNet Conditioning. + .. showcase:: :title: 262-softvc-voice-conversion :img: _static/images/notebook_eye.png @@ -125,6 +147,12 @@ Demos that demonstrate inference on a particular model. Universal segmentation with OneFormer and OpenVINO™. +.. showcase:: + :title: 248-ssd-b1 + :img: https://user-images.githubusercontent.com/29454499/258651862-28b63016-c5ff-4263-9da8-73ca31100165.jpeg + + Image generation with Stable Diffusion XL and OpenVINO™. + .. showcase:: :title: 248-stable-diffusion-xl :img: https://user-images.githubusercontent.com/29454499/258651862-28b63016-c5ff-4263-9da8-73ca31100165.jpeg diff --git a/docs/nbdoc/consts.py b/docs/nbdoc/consts.py index 57d9e603b38..46f8c948025 100644 --- a/docs/nbdoc/consts.py +++ b/docs/nbdoc/consts.py @@ -5,7 +5,7 @@ repo_directory = "notebooks" repo_owner = "openvinotoolkit" repo_name = "openvino_notebooks" repo_branch = "tree/main" -artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/" +artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/" blacklisted_extensions = ['.xml', '.bin'] notebooks_repo = "https://github.com/openvinotoolkit/openvino_notebooks/blob/main/" notebooks_binder = "https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=" diff --git a/docs/notebooks/002-openvino-api-with-output.rst b/docs/notebooks/002-openvino-api-with-output.rst index 1810a90a6ee..466d51c3610 100644 --- a/docs/notebooks/002-openvino-api-with-output.rst +++ b/docs/notebooks/002-openvino-api-with-output.rst @@ -1,8 +1,17 @@ OpenVINO™ Runtime API Tutorial ============================== -This notebook explains the basics of the OpenVINO Runtime API. It -covers: +This notebook explains the basics of the OpenVINO Runtime API. + +The notebook is divided into sections with headers. The next cell +contains global requirements for installation and imports. Each section +is standalone and does not depend on any previous sections. All models +used in this tutorial are provided as examples. These model files can be +replaced with your own models. The exact outputs will be different, but +the process is the same. + +**Table of contents:** + - `Loading OpenVINO Runtime and Showing Info <#loading-openvino-runtime-and-showing-info>`__ @@ -13,6 +22,7 @@ covers: - `PaddlePaddle Model <#paddlepaddle-model>`__ - `TensorFlow Model <#tensorflow-model>`__ - `TensorFlow Lite Model <#tensorflow-lite-model>`__ + - `PyTorch Model <#pytorch-model>`__ - `Getting Information about a Model <#getting-information-about-a-model>`__ @@ -28,19 +38,11 @@ covers: - `Caching a Model <#caching-a-model>`__ -The notebook is divided into sections with headers. The next cell -contains global requirements installation and imports. Each section is -standalone and does not depend on any previous sections. A segmentation -and classification OpenVINO IR model and a segmentation ONNX model are -provided as examples. These model files can be replaced with your own -models. The exact outputs will be different, but the process is the -same. - .. code:: ipython3 # Required imports. Please execute this cell first. - %pip install -q "openvino>=2023.1.0" - %pip install requests tqdm + %pip install -q "openvino>=2023.1.0" + %pip install requests tqdm ipywidgets # Fetch `notebook_utils` module import urllib.request @@ -55,19 +57,44 @@ same. .. parsed-literal:: Note: you may need to restart the kernel to use updated packages. - Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.31.0) - Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.1) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.3.1) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.4) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2.0.7) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2023.7.22) + Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.31.0) + Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.1) + Requirement already satisfied: ipywidgets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (8.1.1) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.3.2) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.6) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2.1.0) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2023.11.17) + Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (0.2.0) + Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (8.12.3) + Requirement already satisfied: traitlets>=4.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (5.14.0) + Requirement already satisfied: widgetsnbextension~=4.0.9 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (4.0.9) + Requirement already satisfied: jupyterlab-widgets~=3.0.9 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (3.0.9) + Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0) + Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1) + Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1) + Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6) + Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5) + Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.41) + Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2) + Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3) + Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0) + Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0) + Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3) + Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0) + Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.12) + Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1) + Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1) + Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2) + Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0) Note: you may need to restart the kernel to use updated packages. Loading OpenVINO Runtime and Showing Info ----------------------------------------- -Initialize OpenVINO Runtime with Core() + + +Initialize OpenVINO Runtime with ``ov.Core()`` .. code:: ipython3 @@ -103,6 +130,8 @@ be faster. Loading a Model --------------- + + After initializing OpenVINO Runtime, first read the model file with ``read_model()``, then compile it to the specified device with the ``compile_model()`` method. @@ -115,6 +144,8 @@ using a tool dedicated to this task. OpenVINO IR Model ~~~~~~~~~~~~~~~~~ + + An OpenVINO IR (Intermediate Representation) model consists of an ``.xml`` file, containing information about network topology, and a ``.bin`` file, containing the weights and biases binary data. Models in @@ -167,7 +198,7 @@ notebooks. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') @@ -184,6 +215,8 @@ notebooks. ONNX Model ~~~~~~~~~~ + + `ONNX `__ is an open format built to represent machine learning models. ONNX defines a common set of operators - the building blocks of machine learning and deep learning models - and a common file @@ -214,7 +247,7 @@ points to the filename of an ONNX model. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx') @@ -237,6 +270,8 @@ The ONNX model can be exported to OpenVINO IR with ``save_model()``: PaddlePaddle Model ~~~~~~~~~~~~~~~~~~ + + `PaddlePaddle `__ models saved for inference can also be passed to OpenVINO Runtime without any conversion step. Pass the filename with extension to @@ -268,7 +303,7 @@ without any conversion step. Pass the filename with extension to .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams') @@ -289,16 +324,10 @@ without any conversion step. Pass the filename with extension to TensorFlow Model ~~~~~~~~~~~~~~~~ -TensorFlow models saved in frozen graph format can also be passed to -``read_model`` starting in OpenVINO 2022.3. - **NOTE**: Directly loading TensorFlow models is available as a - preview feature in the OpenVINO 2022.3 release. Fully functional - support will be provided in the upcoming 2023 releases. Currently - support is limited to only frozen graph inference format. Other - TensorFlow model formats must be converted to OpenVINO IR using - `model conversion - API `__. + +TensorFlow models saved in frozen graph format can also be passed to +``read_model``. .. code:: ipython3 @@ -318,7 +347,7 @@ TensorFlow models saved in frozen graph format can also be passed to .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb') @@ -339,6 +368,8 @@ TensorFlow models saved in frozen graph format can also be passed to TensorFlow Lite Model ~~~~~~~~~~~~~~~~~~~~~ + + `TFLite `__ models saved for inference can also be passed to OpenVINO Runtime. Pass the filename with extension ``.tflite`` to ``read_model`` and exported an OpenVINO IR with @@ -352,7 +383,7 @@ It is pre-trained model optimized to work with TensorFlow Lite. from pathlib import Path - tflite_model_url = 'https://tfhub.dev/tensorflow/lite-model/inception_v4_quant/1/default/1?lite-format=tflite' + tflite_model_url = 'https://www.kaggle.com/models/tensorflow/inception/frameworks/tfLite/variations/v4-quant/versions/1?lite-format=tflite' tflite_model_path = Path('model/classification.tflite') download_file(tflite_model_url, filename=tflite_model_path.name, directory=tflite_model_path.parent) @@ -368,7 +399,7 @@ It is pre-trained model optimized to work with TensorFlow Lite. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite') @@ -385,9 +416,43 @@ It is pre-trained model optimized to work with TensorFlow Lite. ov.save_model(model_tflite, output_model="model/exported_tflite_model.xml") +PyTorch Model +~~~~~~~~~~~~~ + + + +`PyTorch `__ models can not be directly passed to +``core.read_model``. ``ov.Model`` for model objects from this framework +can be obtained using ``ov.convert_model`` API. You can find more +details in `pytorch-to-openvino <../102-pytorch-to-openvino>`__ +notebook. In this tutorial we will use +`resnet18 `__ +model form torchvision library. After conversion model using +``ov.convert_model``, it can be compiled on device using +``core.compile_model`` or saved on disk for the next usage using +``ov.save_model`` + +.. code:: ipython3 + + import openvino as ov + import torch + from torchvision.models import resnet18, ResNet18_Weights + + core = ov.Core() + + pt_model = resnet18(weights=ResNet18_Weights.IMAGENET1K_V1) + example_input = torch.zeros((1, 3, 224, 224)) + ov_model_pytorch = ov.convert_model(pt_model, example_input=example_input) + + compiled_model_pytorch = core.compile_model(ov_model_pytorch, device_name="CPU") + + ov.save_model(ov_model_pytorch, "model/exported_pytorch_model.xml") + Getting Information about a Model --------------------------------- + + The OpenVINO Model instance stores information about the model. Information about the inputs and outputs of the model are in ``model.inputs`` and ``model.outputs``. These are also properties of the @@ -415,13 +480,15 @@ Information about the inputs and outputs of the model are in .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') Model Inputs ~~~~~~~~~~~~ + + Information about all input layers is stored in the ``inputs`` dictionary. @@ -495,6 +562,8 @@ point) precision. Model Outputs ~~~~~~~~~~~~~ + + .. code:: ipython3 import openvino as ov @@ -559,32 +628,41 @@ classes (``C``). The output is returned as 32-bit floating point. Doing Inference on a Model -------------------------- + + **NOTE** this notebook demonstrates only the basic synchronous inference API. For an async inference example, please refer to `Async API notebook <115-async-api-with-output.html>`__ The diagram below shows a typical inference pipeline with OpenVINO -.. figure:: https://docs.openvino.ai/2023.0/_images/IMPLEMENT_PIPELINE_with_API_C.svg +.. figure:: https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/a91bc582-165b-41a2-ab08-12c812059936 :alt: image.png image.png Creating OpenVINO Core and model compilation is covered in the previous -steps. The next step is preparing an inference request. To do inference -on a model, first create an inference request by calling the -``create_infer_request()`` method of ``CompiledModel``, -``compiled_model`` that was loaded with ``compile_model()``. Then, call -the ``infer()`` method of ``InferRequest``. It expects one argument: -``inputs``. This is a dictionary that maps input layer names to input -data or list of input data in ``np.ndarray`` format, where the position -of the input tensor corresponds to input index. If a model has a single -input, wrapping to a dictionary or list can be omitted. +steps. The next step is preparing inputs. You can provide inputs in one +of the supported format: dictionary with name of inputs as keys and +``np.arrays`` that represent input tensors as values, list or tuple of +``np.arrays`` represented input tensors (their order should match with +model inputs order). If a model has a single input, wrapping to a +dictionary or list can be omitted. To do inference on a model, pass +prepared inputs into compiled model object obtained using +``core.compile_model``. The inference result represented as dictionary, +where keys are model outputs and ``np.arrays`` represented their +produced data as values. .. code:: ipython3 # Install opencv package for image handling - !pip install -q opencv-python + %pip install -q opencv-python + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + **Load the network** @@ -608,7 +686,7 @@ input, wrapping to a dictionary or list can be omitted. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') @@ -756,9 +834,13 @@ notebook <001-hello-world-with-output.html>`__. Reshaping and Resizing ---------------------- + + Change Image Size ~~~~~~~~~~~~~~~~~ + + Instead of reshaping the image to fit the model, it is also possible to reshape the model to fit the image. Be aware that not all models support reshaping, and models that do, may not support all input shapes. The @@ -793,7 +875,7 @@ input shape. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin') @@ -847,6 +929,8 @@ dimensions. After reshaping, compile the network once again. Change Batch Size ~~~~~~~~~~~~~~~~~ + + Use the ``.reshape()`` method to set the batch size, by increasing the first element of ``new_shape``. For example, to set a batch size of two, set ``new_shape = (2,3,544,544)`` in the cell above. @@ -907,6 +991,8 @@ input image through the network to see the result: Caching a Model --------------- + + For some devices, like GPU, loading a model can take some time. Model Caching solves this issue by caching the model in a cache directory. If ``core.compile_model(model=net, device_name=device_name, config=config_dict)`` @@ -944,7 +1030,7 @@ the cache. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin') diff --git a/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst b/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst index 81805f59100..2168de355e9 100644 --- a/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst +++ b/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst @@ -44,8 +44,7 @@ documentation `__ - `ONNX Model Conversion <#onnx-model-conversion>`__ - - `Convert PyTorch model to - ONNX <#convert-pytorch-model-to-onnx>`__ + - `Convert PyTorch model to ONNX <#convert-pytorch-model-to-onnx>`__ - `Convert ONNX Model to OpenVINO IR Format <#convert-onnx-model-to-openvino-ir-format>`__ @@ -78,11 +77,15 @@ documentation `__ Note: you may need to restart the kernel to use updated packages. -Preparation ------------------------------------------------------ +Preparation +----------- + + + +Imports +~~~~~~~ + -Imports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 @@ -105,8 +108,10 @@ Imports from notebook_utils import segmentation_map_to_image, viz_result_image, SegmentationMap, Label, download_file -Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Settings +~~~~~~~~ + + Set a name for the model, then define width and height of the image that will be used by the network during inference. According to the input @@ -127,14 +132,14 @@ transforms function, the model is pre-trained on images with a height of onnx_path.parent.mkdir() ir_path = onnx_path.with_suffix(".xml") -Load Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load Model +~~~~~~~~~~ + + Generally, PyTorch models represent an instance of ``torch.nn.Module`` class, initialized by a state dictionary with model weights. Typical -steps for getting a pre-trained model: - -1. Create instance of model class +steps for getting a pre-trained model: 1. Create instance of model class 2. Load checkpoint state dict, which contains pre-trained model weights 3. Turn model to evaluation for switching some operations to inference mode @@ -180,11 +185,15 @@ have not downloaded the model before. Loaded PyTorch LRASPP MobileNetV3 model -ONNX Model Conversion ---------------------------------------------------------------- +ONNX Model Conversion +--------------------- + + + +Convert PyTorch model to ONNX +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + -Convert PyTorch model to ONNX -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ OpenVINO supports PyTorch models that are exported in ONNX format. We will use the ``torch.onnx.export`` function to obtain the ONNX model, @@ -223,8 +232,10 @@ line of the output will read: ONNX model exported to model/lraspp_mobilenet_v3_large.onnx. -Convert ONNX Model to OpenVINO IR Format -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert ONNX Model to OpenVINO IR Format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + To convert the ONNX model to OpenVINO IR with ``FP16`` precision, use model conversion API. The models are saved inside the current directory. @@ -246,14 +257,18 @@ For more information on how to convert models, see this Exporting ONNX model to IR... This may take a few minutes. -Show Results ------------------------------------------------------- +Show Results +------------ + + Confirm that the segmentation results look as expected by comparing model predictions on the ONNX, OpenVINO IR and PyTorch models. -Load and Preprocess an Input Image -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load and Preprocess an Input Image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Images need to be normalized before propagating through the network. @@ -296,8 +311,10 @@ Images need to be normalized before propagating through the network. data/coco.jpg: 0%| | 0.00/202k [00:00`__ - `Pytorch ONNX diff --git a/docs/notebooks/102-pytorch-to-openvino-with-output.rst b/docs/notebooks/102-pytorch-to-openvino-with-output.rst deleted file mode 100644 index 71a724522de..00000000000 --- a/docs/notebooks/102-pytorch-to-openvino-with-output.rst +++ /dev/null @@ -1,796 +0,0 @@ -Convert a PyTorch Model to OpenVINO™ IR -======================================= - -This tutorial demonstrates step-by-step instructions on how to do -inference on a PyTorch classification model using OpenVINO Runtime. -Starting from OpenVINO 2023.0 release, OpenVINO supports direct PyTorch -model conversion without an intermediate step to convert them into ONNX -format. In order, if you try to use the lower OpenVINO version or prefer -to use ONNX, please check this -`tutorial <102-pytorch-to-openvino-with-output.html>`__. - -In this tutorial, we will use the -`RegNetY_800MF `__ model from -`torchvision `__ to -demonstrate how to convert PyTorch models to OpenVINO Intermediate -Representation. - -The RegNet model was proposed in `Designing Network Design -Spaces `__ by Ilija Radosavovic, Raj -Prateek Kosaraju, Ross Girshick, Kaiming He, Piotr Dollár. The authors -design search spaces to perform Neural Architecture Search (NAS). They -first start from a high dimensional search space and iteratively reduce -the search space by empirically applying constraints based on the -best-performing models sampled by the current search space. Instead of -focusing on designing individual network instances, authors design -network design spaces that parametrize populations of networks. The -overall process is analogous to the classic manual design of networks -but elevated to the design space level. The RegNet design space provides -simple and fast networks that work well across a wide range of flop -regimes. - -**Table of contents:** - - -- `Prerequisites <#prerequisites>`__ -- `Load PyTorch Model <#load-pytorch-model>`__ - - - `Prepare Input Data <#prepare-input-data>`__ - - `Run PyTorch Model - Inference <#run-pytorch-model-inference>`__ - - `Benchmark PyTorch Model - Inference <#benchmark-pytorch-model-inference>`__ - -- `Convert PyTorch Model to OpenVINO Intermediate - Representation <#convert-pytorch-model-to-openvino-intermediate-representation>`__ - - - `Select inference device <#select-inference-device>`__ - - `Run OpenVINO Model - Inference <#run-openvino-model-inference>`__ - - `Benchmark OpenVINO Model - Inference <#benchmark-openvino-model-inference>`__ - -- `Convert PyTorch Model with Static Input - Shape <#convert-pytorch-model-with-static-input-shape>`__ - - - `Select inference device <#select-inference-device>`__ - - `Run OpenVINO Model Inference with Static Input - Shape <#run-openvino-model-inference-with-static-input-shape>`__ - - `Benchmark OpenVINO Model Inference with Static Input - Shape <#benchmark-openvino-model-inference-with-static-input-shape>`__ - -- `Convert TorchScript Model to OpenVINO Intermediate - Representation <#convert-torchscript-model-to-openvino-intermediate-representation>`__ - - - `Scripted Model <#scripted-model>`__ - - `Benchmark Scripted Model - Inference <#benchmark-scripted-model-inference>`__ - - `Convert PyTorch Scripted Model to OpenVINO Intermediate - Representation <#convert-pytorch-scripted-model-to-openvino-intermediate-representation>`__ - - `Benchmark OpenVINO Model Inference Converted From Scripted - Model <#benchmark-openvino-model-inference-converted-from-scripted-model>`__ - - `Traced Model <#traced-model>`__ - - `Benchmark Traced Model - Inference <#benchmark-traced-model-inference>`__ - - `Convert PyTorch Traced Model to OpenVINO Intermediate - Representation <#convert-pytorch-traced-model-to-openvino-intermediate-representation>`__ - - `Benchmark OpenVINO Model Inference Converted From Traced - Model <#benchmark-openvino-model-inference-converted-from-traced-model>`__ - -Prerequisites -------------------------------------------------------- - -Install notebook dependencies - -.. code:: ipython3 - - %pip install -q "openvino>=2023.1.0" scipy - - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - - -Download input data and label map - -.. code:: ipython3 - - import requests - from pathlib import Path - from PIL import Image - - MODEL_DIR = Path("model") - DATA_DIR = Path("data") - - MODEL_DIR.mkdir(exist_ok=True) - DATA_DIR.mkdir(exist_ok=True) - MODEL_NAME = "regnet_y_800mf" - - image = Image.open(requests.get("https://farm9.staticflickr.com/8225/8511402100_fea15da1c5_z.jpg", stream=True).raw) - - labels_file = DATA_DIR / "imagenet_2012.txt" - - if not labels_file.exists(): - resp = requests.get("https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/imagenet_2012.txt") - with labels_file.open("wb") as f: - f.write(resp.content) - - imagenet_classes = labels_file.open("r").read().splitlines() - -Load PyTorch Model ------------------------------------------------------------- - -Generally, PyTorch models represent an instance of the -``torch.nn.Module`` class, initialized by a state dictionary with model -weights. Typical steps for getting a pre-trained model: - -1. Create an instance of a model class -2. Load checkpoint state dict, which contains pre-trained model weights -3. Turn the model to evaluation for switching some operations to - inference mode - -The ``torchvision`` module provides a ready-to-use set of functions for -model class initialization. We will use -``torchvision.models.regnet_y_800mf``. You can directly pass pre-trained -model weights to the model initialization function using the weights -enum ``RegNet_Y_800MF_Weights.DEFAULT``. - -.. code:: ipython3 - - import torchvision - - # get default weights using available weights Enum for model - weights = torchvision.models.RegNet_Y_800MF_Weights.DEFAULT - - # create model topology and load weights - model = torchvision.models.regnet_y_800mf(weights=weights) - - # switch model to inference mode - model.eval(); - -Prepare Input Data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The code below demonstrates how to preprocess input data using a -model-specific transforms module from ``torchvision``. After -transformation, we should concatenate images into batched tensor, in our -case, we will run the model with batch 1, so we just unsqueeze input on -the first dimension. - -.. code:: ipython3 - - import torch - - # Initialize the Weight Transforms - preprocess = weights.transforms() - - # Apply it to the input image - img_transformed = preprocess(image) - - # Add batch dimension to image tensor - input_tensor = img_transformed.unsqueeze(0) - -Run PyTorch Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The model returns a vector of probabilities in raw logits format, -softmax can be applied to get normalized values in the [0, 1] range. For -a demonstration that the output of the original model and OpenVINO -converted is the same, we defined a common postprocessing function which -can be reused later. - -.. code:: ipython3 - - import numpy as np - from scipy.special import softmax - - # Perform model inference on input tensor - result = model(input_tensor) - - # Postprocessing function for getting results in the same way for both PyTorch model inference and OpenVINO - def postprocess_result(output_tensor:np.ndarray, top_k:int = 5): - """ - Posprocess model results. This function applied sofrmax on output tensor and returns specified top_k number of labels with highest probability - Parameters: - output_tensor (np.ndarray): model output tensor with probabilities - top_k (int, *optional*, default 5): number of labels with highest probability for return - Returns: - topk_labels: label ids for selected top_k scores - topk_scores: selected top_k highest scores predicted by model - """ - softmaxed_scores = softmax(output_tensor, -1)[0] - topk_labels = np.argsort(softmaxed_scores)[-top_k:][::-1] - topk_scores = softmaxed_scores[topk_labels] - return topk_labels, topk_scores - - # Postprocess results - top_labels, top_scores = postprocess_result(result.detach().numpy()) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_11_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark PyTorch Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - # Run model inference - model(input_tensor) - - -.. parsed-literal:: - - 17.5 ms ± 9.66 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - - -Convert PyTorch Model to OpenVINO Intermediate Representation -------------------------------------------------------------------------------------------------------- - -Starting from the 2023.0 release OpenVINO supports direct PyTorch models -conversion to OpenVINO Intermediate Representation (IR) format. OpenVINO -model conversion API should be used for these purposes. More details -regarding PyTorch model conversion can be found in OpenVINO -`documentation `__ - - **Note**: Please, take into account that direct support PyTorch - models conversion is an experimental feature. Model coverage will be - increased in the next releases. For cases, when PyTorch model - conversion failed, you still can try to export the model to ONNX - format. Please refer to this - `tutorial <102-pytorch-to-openvino-with-output.html>`__ - which explains how to convert PyTorch model to ONNX, then to OpenVINO - -The ``convert_model`` function accepts the PyTorch model object and -returns the ``openvino.Model`` instance ready to load on a device using -``core.compile_model`` or save on disk for next usage using -``ov.save_model``. Optionally, we can provide additional parameters, -such as: - -- ``compress_to_fp16`` - flag to perform model weights compression into - FP16 data format. It may reduce the required space for model storage - on disk and give speedup for inference devices, where FP16 - calculation is supported. -- ``example_input`` - input data sample which can be used for model - tracing. -- ``input_shape`` - the shape of input tensor for conversion - -and any other advanced options supported by model conversion Python API. -More details can be found on this -`page `__ - -.. code:: ipython3 - - import openvino as ov - - # Create OpenVINO Core object instance - core = ov.Core() - - # Convert model to openvino.runtime.Model object - ov_model = ov.convert_model(model) - - # Save openvino.runtime.Model object on disk - ov.save_model(ov_model, MODEL_DIR / f"{MODEL_NAME}_dynamic.xml") - - ov_model - - - - -.. parsed-literal:: - - - ] - outputs[ - - ]> - - - -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -select device from dropdown list for running inference using OpenVINO - -.. code:: ipython3 - - import ipywidgets as widgets - - device = widgets.Dropdown( - options=core.available_devices + ["AUTO"], - value='AUTO', - description='Device:', - disabled=False, - ) - - device - - - - -.. parsed-literal:: - - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') - - - -.. code:: ipython3 - - # Load OpenVINO model on device - compiled_model = core.compile_model(ov_model, device.value) - compiled_model - - - - -.. parsed-literal:: - - - ] - outputs[ - - ]> - - - -Run OpenVINO Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - # Run model inference - result = compiled_model(input_tensor)[0] - - # Posptorcess results - top_labels, top_scores = postprocess_result(result) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_20_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark OpenVINO Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - compiled_model(input_tensor) - - -.. parsed-literal:: - - 3.21 ms ± 12 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - - -Convert PyTorch Model with Static Input Shape ---------------------------------------------------------------------------------------- - -The default conversion path preserves dynamic input shapes, in order if -you want to convert the model with static shapes, you can explicitly -specify it during conversion using the ``input_shape`` parameter or -reshape the model into the desired shape after conversion. For the model -reshaping example please check the following -`tutorial <002-openvino-api-with-output.html>`__. - -.. code:: ipython3 - - # Convert model to openvino.runtime.Model object - ov_model = ov.convert_model(model, input=[[1,3,224,224]]) - # Save openvino.runtime.Model object on disk - ov.save_model(ov_model, MODEL_DIR / f"{MODEL_NAME}_static.xml") - ov_model - - - - -.. parsed-literal:: - - - ] - outputs[ - - ]> - - - -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -select device from dropdown list for running inference using OpenVINO - -.. code:: ipython3 - - device - - - - -.. parsed-literal:: - - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') - - - -.. code:: ipython3 - - # Load OpenVINO model on device - compiled_model = core.compile_model(ov_model, device.value) - compiled_model - - - - -.. parsed-literal:: - - - ] - outputs[ - - ]> - - - -Now, we can see that input of our converted model is tensor of shape [1, -3, 224, 224] instead of [?, 3, ?, ?] reported by previously converted -model. - -Run OpenVINO Model Inference with Static Input Shape -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - # Run model inference - result = compiled_model(input_tensor)[0] - - # Posptorcess results - top_labels, top_scores = postprocess_result(result) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_31_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark OpenVINO Model Inference with Static Input Shape -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - compiled_model(input_tensor) - - -.. parsed-literal:: - - 2.79 ms ± 12 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - - -Convert TorchScript Model to OpenVINO Intermediate Representation ------------------------------------------------------------------------------------------------------------ - -TorchScript is a way to create serializable and optimizable models from -PyTorch code. Any TorchScript program can be saved from a Python process -and loaded in a process where there is no Python dependency. More -details about TorchScript can be found in `PyTorch -documentation `__. - -There are 2 possible ways to convert the PyTorch model to TorchScript: - -- ``torch.jit.script`` - Scripting a function or ``nn.Module`` will - inspect the source code, compile it as TorchScript code using the - TorchScript compiler, and return a ``ScriptModule`` or - ``ScriptFunction``. -- ``torch.jit.trace`` - Trace a function and return an executable or - ``ScriptFunction`` that will be optimized using just-in-time - compilation. - -Let’s consider both approaches and their conversion into OpenVINO IR. - -Scripted Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``torch.jit.script`` inspects model source code and compiles it to -``ScriptModule``. After compilation model can be used for inference or -saved on disk using the ``torch.jit.save`` function and after that -restored with ``torch.jit.load`` in any other environment without the -original PyTorch model code definitions. - -TorchScript itself is a subset of the Python language, so not all -features in Python work, but TorchScript provides enough functionality -to compute on tensors and do control-dependent operations. For a -complete guide, see the `TorchScript Language -Reference `__. - -.. code:: ipython3 - - # Get model path - scripted_model_path = MODEL_DIR / f"{MODEL_NAME}_scripted.pth" - - # Compile and save model if it has not been compiled before or load compiled model - if not scripted_model_path.exists(): - scripted_model = torch.jit.script(model) - torch.jit.save(scripted_model, scripted_model_path) - else: - scripted_model = torch.jit.load(scripted_model_path) - - # Run scripted model inference - result = scripted_model(input_tensor) - - # Postprocess results - top_labels, top_scores = postprocess_result(result.detach().numpy()) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_35_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark Scripted Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - scripted_model(input_tensor) - - -.. parsed-literal:: - - 12.9 ms ± 9.28 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - - -Convert PyTorch Scripted Model to OpenVINO Intermediate Representation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The conversion step for the scripted model to OpenVINO IR is similar to -the original PyTorch model. - -.. code:: ipython3 - - # Convert model to openvino.runtime.Model object - ov_model = ov.convert_model(scripted_model) - - # Load OpenVINO model on device - compiled_model = core.compile_model(ov_model, device.value) - - # Run OpenVINO model inference - result = compiled_model(input_tensor, device.value)[0] - - # Postprocess results - top_labels, top_scores = postprocess_result(result) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_39_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark OpenVINO Model Inference Converted From Scripted Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - compiled_model(input_tensor) - - -.. parsed-literal:: - - 3.21 ms ± 17.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - - -Traced Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Using ``torch.jit.trace``, you can turn an existing module or Python -function into a TorchScript ``ScriptFunction`` or ``ScriptModule``. You -must provide example inputs, and model will be executed, recording the -operations performed on all the tensors. - -- The resulting recording of a standalone function produces - ``ScriptFunction``. - -- The resulting recording of ``nn.Module.forward`` or ``nn.Module`` - produces ``ScriptModule``. - -In the same way like scripted model, traced model can be used for -inference or saved on disk using ``torch.jit.save`` function and after -that restored with ``torch.jit.load`` in any other environment without -original PyTorch model code definitions. - -.. code:: ipython3 - - # Get model path - traced_model_path = MODEL_DIR / f"{MODEL_NAME}_traced.pth" - - # Trace and save model if it has not been traced before or load traced model - if not traced_model_path.exists(): - traced_model = torch.jit.trace(model, example_inputs=input_tensor) - torch.jit.save(traced_model, traced_model_path) - else: - traced_model = torch.jit.load(traced_model_path) - - # Run traced model inference - result = traced_model(input_tensor) - - # Postprocess results - top_labels, top_scores = postprocess_result(result.detach().numpy()) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_43_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark Traced Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - traced_model(input_tensor) - - -.. parsed-literal:: - - 13.4 ms ± 22.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - - -Convert PyTorch Traced Model to OpenVINO Intermediate Representation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The conversion step for a traced model to OpenVINO IR is similar to the -original PyTorch model. - -.. code:: ipython3 - - # Convert model to openvino.runtime.Model object - ov_model = ov.convert_model(traced_model) - - # Load OpenVINO model on device - compiled_model = core.compile_model(ov_model, device.value) - - # Run OpenVINO model inference - result = compiled_model(input_tensor)[0] - - # Postprocess results - top_labels, top_scores = postprocess_result(result) - - # Show results - display(image) - for idx, (label, score) in enumerate(zip(top_labels, top_scores)): - _, predicted_label = imagenet_classes[label].split(" ", 1) - print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%") - - - -.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_47_0.png - - -.. parsed-literal:: - - 1: tiger cat - 25.91% - 2: Egyptian cat - 10.26% - 3: computer keyboard, keypad - 9.22% - 4: tabby, tabby cat - 9.09% - 5: hamper - 2.35% - - -Benchmark OpenVINO Model Inference Converted From Traced Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - %%timeit - - compiled_model(input_tensor)[0] - - -.. parsed-literal:: - - 2.82 ms ± 8.37 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - diff --git a/docs/notebooks/102-pytorch-to-openvino-with-output_files/index.html b/docs/notebooks/102-pytorch-to-openvino-with-output_files/index.html index 2b4be31be6c..953c8eb9d09 100644 --- a/docs/notebooks/102-pytorch-to-openvino-with-output_files/index.html +++ b/docs/notebooks/102-pytorch-to-openvino-with-output_files/index.html @@ -1,20 +1,20 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/102-pytorch-to-openvino-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/102-pytorch-to-openvino-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/102-pytorch-to-openvino-with-output_files/


../
-102-pytorch-to-openvino-with-output_11_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_11_0.png       31-Oct-2023 00:35              542516
-102-pytorch-to-openvino-with-output_20_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_20_0.png       31-Oct-2023 00:35              542516
-102-pytorch-to-openvino-with-output_31_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_31_0.png       31-Oct-2023 00:35              542516
-102-pytorch-to-openvino-with-output_35_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_35_0.png       31-Oct-2023 00:35              542516
-102-pytorch-to-openvino-with-output_39_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_39_0.png       31-Oct-2023 00:35              542516
-102-pytorch-to-openvino-with-output_43_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_43_0.png       31-Oct-2023 00:35              542516
-102-pytorch-to-openvino-with-output_47_0.jpg       31-Oct-2023 00:35               54874
-102-pytorch-to-openvino-with-output_47_0.png       31-Oct-2023 00:35              542516
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/102-pytorch-to-openvino-with-output_files/


../
+102-pytorch-to-openvino-with-output_11_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_11_0.png       07-Dec-2023 00:49              542516
+102-pytorch-to-openvino-with-output_20_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_20_0.png       07-Dec-2023 00:49              542516
+102-pytorch-to-openvino-with-output_31_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_31_0.png       07-Dec-2023 00:49              542516
+102-pytorch-to-openvino-with-output_35_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_35_0.png       07-Dec-2023 00:49              542516
+102-pytorch-to-openvino-with-output_39_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_39_0.png       07-Dec-2023 00:49              542516
+102-pytorch-to-openvino-with-output_43_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_43_0.png       07-Dec-2023 00:49              542516
+102-pytorch-to-openvino-with-output_47_0.jpg       07-Dec-2023 00:49               54874
+102-pytorch-to-openvino-with-output_47_0.png       07-Dec-2023 00:49              542516
 

diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output.rst b/docs/notebooks/103-paddle-to-openvino-classification-with-output.rst index 50ec3305ec9..53d9144842e 100644 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output.rst +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output.rst @@ -33,42 +33,35 @@ Source of the - `Select inference device <#select-inference-device>`__ - `References <#references>`__ -Preparation ------------------------------------------------------ - -Imports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: ipython3 - - import sys - - if sys.version_info.minor > 7: - %pip install -q "paddlepaddle>=2.5.1" - else: - %pip install -q "paddlepaddle==2.4.2" +Preparation +----------- -.. parsed-literal:: - Note: you may need to restart the kernel to use updated packages. +Imports +~~~~~~~ + .. code:: ipython3 - !pip install -q paddleclas --no-deps - !pip install -q "prettytable" "ujson" "visualdl>=2.2.0" "faiss-cpu>=1.7.1" + %pip install -q "paddlepaddle>=2.5.1" + %pip install -q paddleclas --no-deps + %pip install -q "prettytable" "ujson" "visualdl>=2.2.0" "faiss-cpu>=1.7.1" # Install openvino package !pip install -q "openvino>=2023.1.0" .. parsed-literal:: + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. paddleclas 2.5.1 requires easydict, which is not installed. paddleclas 2.5.1 requires faiss-cpu==1.7.1.post2, but you have faiss-cpu 1.7.4 which is incompatible. paddleclas 2.5.1 requires gast==0.3.3, but you have gast 0.4.0 which is incompatible. - + Note: you may need to restart the kernel to use updated packages. + .. code:: ipython3 @@ -81,11 +74,11 @@ Imports .. parsed-literal:: - --2023-10-30 22:31:22-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb + --2023-12-06 22:32:58-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb Resolving proxy-mu.intel.com (proxy-mu.intel.com)... 10.217.247.236 Connecting to proxy-mu.intel.com (proxy-mu.intel.com)|10.217.247.236|:911... connected. Proxy request sent, awaiting response... 404 Not Found - 2023-10-30 22:31:22 ERROR 404: Not Found. + 2023-12-06 22:32:59 ERROR 404: Not Found. dpkg: error: cannot access archive 'libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb': No such file or directory @@ -114,12 +107,14 @@ Imports .. parsed-literal:: - 2023-10-30 22:31:24 INFO: Loading faiss with AVX2 support. - 2023-10-30 22:31:24 INFO: Successfully loaded faiss with AVX2 support. + 2023-12-06 22:33:00 INFO: Loading faiss with AVX2 support. + 2023-12-06 22:33:00 INFO: Successfully loaded faiss with AVX2 support. + + +Settings +~~~~~~~~ -Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Set ``IMAGE_FILENAME`` to the filename of an image to use. Set ``MODEL_NAME`` to the PaddlePaddle model to download from PaddleHub. @@ -175,8 +170,10 @@ PaddleHub. This may take a while. Model Extracted to "./model". -Show Inference on PaddlePaddle Model ------------------------------------------------------------------------------- +Show Inference on PaddlePaddle Model +------------------------------------ + + In the next cell, we load the model, load and display an image, do inference on that image, and then show the top three prediction results. @@ -195,7 +192,7 @@ inference on that image, and then show the top three prediction results. .. parsed-literal:: - [2023/10/30 22:31:43] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead. + [2023/12/06 22:33:21] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead. Labrador retriever, 0.75138 German short-haired pointer, 0.02373 Great Dane, 0.01848 @@ -204,7 +201,7 @@ inference on that image, and then show the top three prediction results. -.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_9_1.png +.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png ``classifier.predict()`` takes an image file name, reads the image, @@ -259,26 +256,26 @@ clipping values. plt.imshow(np.transpose(processed_image, (1, 2, 0))) +.. parsed-literal:: + + 2023-12-06 22:33:22 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). + + .. parsed-literal:: Processed image shape: (3, 224, 224) -.. parsed-literal:: - - 2023-10-30 22:31:44 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). - - .. parsed-literal:: - + -.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_16_3.png +.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_15_3.png To decode the labels predicted by the model to names of classes, we need @@ -297,8 +294,10 @@ OpenVINO model. partition = line.split("\n")[0].partition(" ") class_id_map[int(partition[0])] = str(partition[-1]) -Convert the Model to OpenVINO IR Format ---------------------------------------------------------------------------------- +Convert the Model to OpenVINO IR Format +--------------------------------------- + + Call the OpenVINO Model Conversion API to convert the PaddlePaddle model to OpenVINO IR, with FP32 precision. ``ov.convert_model`` function @@ -318,8 +317,10 @@ for more information about the Model Conversion API. else: print(f"{model_xml} already exists.") -Select inference device ------------------------------------------------------------------ +Select inference device +----------------------- + + select device from dropdown list for running inference using OpenVINO @@ -346,8 +347,10 @@ select device from dropdown list for running inference using OpenVINO -Show Inference on OpenVINO Model --------------------------------------------------------------------------- +Show Inference on OpenVINO Model +-------------------------------- + + Load the IR model, get model information, load the image, do inference, convert the inference to a meaningful result, and show the output. See @@ -391,11 +394,13 @@ information. -.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_24_1.png +.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png + + +Timing and Comparison +--------------------- -Timing and Comparison ---------------------------------------------------------------- Measure the time it takes to do inference on fifty images and compare the result. The timing information gives an indication of performance. @@ -448,7 +453,7 @@ Note that many optimizations are possible to improve the performance. .. parsed-literal:: - PaddlePaddle model on CPU: 0.0071 seconds per image, FPS: 141.73 + PaddlePaddle model on CPU: 0.0070 seconds per image, FPS: 142.41 PaddlePaddle result: Labrador retriever, 0.75138 @@ -459,11 +464,13 @@ Note that many optimizations are possible to improve the performance. -.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_28_1.png +.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png + + +Select inference device +----------------------- -Select inference device ------------------------------------------------------------------ select device from dropdown list for running inference using OpenVINO @@ -510,7 +517,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0030 seconds per image, FPS: 328.24 + OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0028 seconds per image, FPS: 352.29 OpenVINO result: Labrador retriever, 0.74909 @@ -521,11 +528,13 @@ select device from dropdown list for running inference using OpenVINO -.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_31_1.png +.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png + + +References +---------- -References ----------------------------------------------------- - `PaddleClas `__ - `OpenVINO PaddlePaddle diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_15_3.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_15_3.png new file mode 100644 index 00000000000..87f3978ae62 --- /dev/null +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_15_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37fe65815997e6f67c6a635a98caf7ad3d5066aee57709f650fe8ef4c8bdfe11 +size 120883 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_16_3.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_16_3.png deleted file mode 100644 index 9928929e2f6..00000000000 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_16_3.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:306bb006db6e8ef4b71e12f1007cf62a782408867b1c5e2af981d0dcde7d50e7 -size 120883 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png new file mode 100644 index 00000000000..87b70a6bf1d --- /dev/null +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5 +size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_24_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_24_1.png deleted file mode 100644 index 1e378b726a5..00000000000 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_24_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aaa7ff0a118fe7ac95479e0467f34f793d1013d972c5c850c610e39f6983ee3c -size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png new file mode 100644 index 00000000000..87b70a6bf1d --- /dev/null +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5 +size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_28_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_28_1.png deleted file mode 100644 index 1e378b726a5..00000000000 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_28_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aaa7ff0a118fe7ac95479e0467f34f793d1013d972c5c850c610e39f6983ee3c -size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png new file mode 100644 index 00000000000..87b70a6bf1d --- /dev/null +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5 +size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_31_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_31_1.png deleted file mode 100644 index 1e378b726a5..00000000000 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_31_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aaa7ff0a118fe7ac95479e0467f34f793d1013d972c5c850c610e39f6983ee3c -size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png new file mode 100644 index 00000000000..87b70a6bf1d --- /dev/null +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5 +size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_9_1.png b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_9_1.png deleted file mode 100644 index 1e378b726a5..00000000000 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_9_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aaa7ff0a118fe7ac95479e0467f34f793d1013d972c5c850c610e39f6983ee3c -size 224886 diff --git a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html index 8b1207e9e33..66c78f6a2a4 100644 --- a/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html +++ b/docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html @@ -1,11 +1,11 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/


../
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35              120883
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35              224886
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35              224886
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35              224886
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35              224886
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/


../
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49              120883
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49              224886
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49              224886
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49              224886
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49              224886
 

diff --git a/docs/notebooks/105-language-quantize-bert-with-output.rst b/docs/notebooks/105-language-quantize-bert-with-output.rst index e3fddbbfb9c..5c089756cf1 100644 --- a/docs/notebooks/105-language-quantize-bert-with-output.rst +++ b/docs/notebooks/105-language-quantize-bert-with-output.rst @@ -31,8 +31,7 @@ and datasets. It consists of the following steps: - `Prepare the Dataset <#prepare-the-dataset>`__ - `Optimize model using NNCF Post-training Quantization API <#optimize-model-using-nncf-post-training-quantization-api>`__ -- `Load and Test OpenVINO - Model <#load-and-test-openvino-model>`__ +- `Load and Test OpenVINO Model <#load-and-test-openvino-model>`__ - `Select inference device <#select-inference-device>`__ @@ -44,7 +43,7 @@ and datasets. It consists of the following steps: .. code:: ipython3 %pip install -q "nncf>=2.5.0" - %pip install -q "transformers" datasets evaluate + %pip install -q "transformers" datasets evaluate --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "openvino>=2023.1.0" @@ -55,8 +54,10 @@ and datasets. It consists of the following steps: Note: you may need to restart the kernel to use updated packages. -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -87,10 +88,10 @@ Imports .. parsed-literal:: - 2023-10-30 22:33:08.247649: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-30 22:33:08.281400: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-06 22:34:55.977192: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 22:34:56.010680: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-30 22:33:08.912908: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-06 22:34:56.639162: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -98,8 +99,10 @@ Imports INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -Settings --------------------------------------------------- +Settings +-------- + + .. code:: ipython3 @@ -113,8 +116,10 @@ Settings os.makedirs(DATA_DIR, exist_ok=True) os.makedirs(MODEL_DIR, exist_ok=True) -Prepare the Model ------------------------------------------------------------ +Prepare the Model +----------------- + + Perform the following: @@ -180,22 +185,13 @@ PyTorch model formats are supported: .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - - -.. parsed-literal:: - - WARNING:nncf:NNCF provides best results with torch==2.0.1, while current torch version is 2.1.0+cpu. If you encounter issues, consider switching to torch==2.0.1 - - -.. parsed-literal:: - No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/annotations.py:386: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either. - warnings.warn( -Prepare the Dataset -------------------------------------------------------------- +Prepare the Dataset +------------------- + + We download the `General Language Understanding Evaluation (GLUE) `__ dataset for the MRPC task from @@ -219,15 +215,10 @@ tokenizer from HuggingFace. data_source = create_data_source() +Optimize model using NNCF Post-training Quantization API +-------------------------------------------------------- -.. parsed-literal:: - - Map: 0%| | 0/408 [00:00`__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -261,21 +252,93 @@ The optimization process contains the following steps: model_type=ModelType.TRANSFORMER) + .. parsed-literal:: - Statistics collection: 100%|██████████| 300/300 [00:07<00:00, 39.50it/s] - Applying Smooth Quant: 100%|██████████| 50/50 [00:00<00:00, 51.91it/s] + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + .. parsed-literal:: - INFO:nncf:36 ignored nodes was found by name in the NNCFGraph + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ .. parsed-literal:: - Statistics collection: 100%|██████████| 300/300 [00:25<00:00, 11.96it/s] - Applying Fast Bias correction: 100%|██████████| 74/74 [00:25<00:00, 2.93it/s] + INFO:nncf:36 ignored nodes were found by name in the NNCFGraph + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ .. code:: ipython3 @@ -283,8 +346,10 @@ The optimization process contains the following steps: compressed_model_xml = Path(MODEL_DIR) / "quantized_bert_mrpc.xml" ov.save_model(quantized_model, compressed_model_xml) -Load and Test OpenVINO Model ----------------------------------------------------------------------- +Load and Test OpenVINO Model +---------------------------- + + To load and test converted model, perform the following: @@ -293,8 +358,10 @@ To load and test converted model, perform the following: - Run the inference. - Get the answer from the model output. -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -352,8 +419,10 @@ changing ``sample_idx`` to another value (from 0 to 407). The same meaning: yes -Compare F1-score of FP32 and INT8 models ----------------------------------------------------------------------------------- +Compare F1-score of FP32 and INT8 models +---------------------------------------- + + .. code:: ipython3 @@ -393,11 +462,13 @@ Compare F1-score of FP32 and INT8 models Checking the accuracy of the original model: F1 score: 0.9019 Checking the accuracy of the quantized model: - F1 score: 0.8985 + F1 score: 0.8969 + + +Compare Performance of the Original, Converted and Quantized Models +------------------------------------------------------------------- -Compare Performance of the Original, Converted and Quantized Models -------------------------------------------------------------------------------------------------------------- Compare the original PyTorch model with OpenVINO converted and quantized models (``FP32``, ``INT8``) to see the difference in performance. It is @@ -454,9 +525,9 @@ Frames Per Second (FPS) for images. .. parsed-literal:: - PyTorch model on CPU: 0.073 seconds per sentence, SPS: 13.72 - IR FP32 model in OpenVINO Runtime/AUTO: 0.022 seconds per sentence, SPS: 46.40 - OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.010 seconds per sentence, SPS: 98.65 + PyTorch model on CPU: 0.073 seconds per sentence, SPS: 13.77 + IR FP32 model in OpenVINO Runtime/AUTO: 0.021 seconds per sentence, SPS: 48.61 + OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 109.06 Finally, measure the inference performance of OpenVINO ``FP32`` and @@ -486,20 +557,21 @@ in OpenVINO. [Step 2/11] Loading OpenVINO Runtime [ WARNING ] Default duration 120 seconds is used for unknown device device.value [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: - [ ERROR ] Exception from src/inference/src/core.cpp:84: - Exception from src/inference/src/dev/core_impl.cpp:565: + [ INFO ] + [ INFO ] + [Step 3/11] Setting device configuration + [ ERROR ] Exception from src/inference/src/core.cpp:244: + Exception from src/inference/src/dev/core_impl.cpp:559: Device with "device" name is not registered in the OpenVINO Runtime Traceback (most recent call last): - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 102, in main - benchmark.print_version_info() - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info - for device, version in self.core.get_versions(self.device).items(): - RuntimeError: Exception from src/inference/src/core.cpp:84: - Exception from src/inference/src/dev/core_impl.cpp:565: + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 165, in main + supported_properties = benchmark.core.get_property(device, properties.supported_properties()) + RuntimeError: Exception from src/inference/src/core.cpp:244: + Exception from src/inference/src/dev/core_impl.cpp:559: Device with "device" name is not registered in the OpenVINO Runtime @@ -518,20 +590,21 @@ in OpenVINO. [Step 2/11] Loading OpenVINO Runtime [ WARNING ] Default duration 120 seconds is used for unknown device device.value [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: - [ ERROR ] Exception from src/inference/src/core.cpp:84: - Exception from src/inference/src/dev/core_impl.cpp:565: + [ INFO ] + [ INFO ] + [Step 3/11] Setting device configuration + [ ERROR ] Exception from src/inference/src/core.cpp:244: + Exception from src/inference/src/dev/core_impl.cpp:559: Device with "device" name is not registered in the OpenVINO Runtime Traceback (most recent call last): - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 102, in main - benchmark.print_version_info() - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info - for device, version in self.core.get_versions(self.device).items(): - RuntimeError: Exception from src/inference/src/core.cpp:84: - Exception from src/inference/src/dev/core_impl.cpp:565: + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 165, in main + supported_properties = benchmark.core.get_property(device, properties.supported_properties()) + RuntimeError: Exception from src/inference/src/core.cpp:244: + Exception from src/inference/src/dev/core_impl.cpp:559: Device with "device" name is not registered in the OpenVINO Runtime diff --git a/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst b/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst index 1173f4b74c3..77ad8fdb5d1 100644 --- a/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst +++ b/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst @@ -30,8 +30,7 @@ steps: Representation <#convert-model-to-openvino-intermediate-representation>`__ - `Prepare inference data <#prepare-inference-data>`__ -- `Check model inference - result <#check-model-inference-result>`__ +- `Check model inference result <#check-model-inference-result>`__ - `Validate model accuracy on dataset <#validate-model-accuracy-on-dataset>`__ - `Quantization <#quantization>`__ @@ -42,8 +41,10 @@ steps: - `Compare Accuracy of the Original and Quantized Models <#compare-accuracy-of-the-original-and-quantized-models>`__ -Download and prepare model --------------------------------------------------------------------- +Download and prepare model +-------------------------- + + data2vec is a framework for self-supervised representation learning for images, speech, and text as described in `data2vec: A General Framework @@ -61,8 +62,10 @@ In our case, we will use ``data2vec-audio-base-960h`` model, which was finetuned on 960 hours of audio from LibriSpeech Automatic Speech Recognition corpus and distributed as part of HuggingFace transformers. -Obtain Pytorch model representation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Obtain Pytorch model representation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + For instantiating PyTorch model class, we should use ``Data2VecAudioForCTC.from_pretrained`` method with providing model ID @@ -77,8 +80,8 @@ model specific pre- and post-processing steps. .. code:: ipython3 %pip install -q "openvino>=2023.1.0" "nncf>=2.5.0" - %pip install -q datasets "torchmetrics>=0.11.0" - %pip install -q soundfile librosa transformers + %pip install -q datasets "torchmetrics>=0.11.0" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q soundfile librosa transformers --extra-index-url https://download.pytorch.org/whl/cpu .. code:: ipython3 @@ -96,8 +99,10 @@ model specific pre- and post-processing steps. 2023-09-12 19:27:58.411557: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT -Convert model to OpenVINO Intermediate Representation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert model to OpenVINO Intermediate Representation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -133,8 +138,10 @@ Convert model to OpenVINO Intermediate Representation Read IR model from model/data2vec-audo-base.xml -Prepare inference data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare inference data +~~~~~~~~~~~~~~~~~~~~~~ + + For demonstration purposes, we will use short dummy version of LibriSpeech dataset - ``patrickvonplaten/librispeech_asr_dummy`` to @@ -169,8 +176,10 @@ dataset. Loading cached processed dataset at /home/ea/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc/cache-5282243604a7a526.arrow -Check model inference result ----------------------------------------------------------------------- +Check model inference result +---------------------------- + + The code below is used for running model inference on a single sample from the dataset. It contains the following steps: @@ -242,8 +251,10 @@ For reference, see the same function provided for OpenVINO model. -Validate model accuracy on dataset ----------------------------------------------------------------------------- +Validate model accuracy on dataset +---------------------------------- + + For model accuracy evaluation, `Word Error Rate `__ metric can be @@ -302,8 +313,10 @@ library. [OpenVino] Word Error Rate: 0.0383 -Quantization ------------------------------------------------------- +Quantization +------------ + + `NNCF `__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -565,8 +578,10 @@ saved using ``ov.save_model`` function. quantized_model_path = Path(f"{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml") ov.save_model(quantized_model, quantized_model_path) -Check INT8 model inference result ---------------------------------------------------------------------------- +Check INT8 model inference result +--------------------------------- + + ``INT8`` model is the same in usage like the original one. We need to read it, using the ``core.read_model`` method and load on the device, @@ -604,8 +619,8 @@ using ``core.compile_model``. After that, we can reuse the same -Compare Performance of the Original and Quantized Models --------------------------------------------------------------------------------------------------- +Compare Performance of the Original and Quantized Models +-------------------------------------------------------- `Benchmark Tool `__ @@ -771,8 +786,10 @@ is used to measure the inference performance of the ``FP16`` and [ INFO ] Throughput: 66.40 FPS -Compare Accuracy of the Original and Quantized Models ------------------------------------------------------------------------------------------------ +Compare Accuracy of the Original and Quantized Models +----------------------------------------------------- + + Finally, calculate WER metric for the ``INT8`` model representation and compare it with the ``FP16`` result. diff --git a/docs/notebooks/109-latency-tricks-with-output.rst b/docs/notebooks/109-latency-tricks-with-output.rst index 60ba8c7a7d2..4d93f99b2be 100644 --- a/docs/notebooks/109-latency-tricks-with-output.rst +++ b/docs/notebooks/109-latency-tricks-with-output.rst @@ -27,8 +27,8 @@ The quantization and pre-post-processing API are not included here as they change the precision (quantization) or processing graph (prepostprocessor). You can find examples of how to apply them to optimize performance on OpenVINO IR files in -`111-detection-quantization <111-detection-quantization-with-output.html>`__ and -`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__. +`111-detection-quantization <../111-detection-quantization>`__ and +`118-optimize-preprocessing <../118-optimize-preprocessing>`__. |image0| @@ -44,11 +44,12 @@ optimize performance on OpenVINO IR files in result in different performance. A similar notebook focused on the throughput mode is available -`here <109-throughput-tricks-with-output.html>`__. +`here <109-throughput-tricks.ipynb>`__. **Table of contents:** +- `Prerequisites <#prerequisites>`__ - `Data <#data>`__ - `Model <#model>`__ - `Hardware <#hardware>`__ @@ -70,14 +71,16 @@ A similar notebook focused on the throughput mode is available - `Performance comparison <#performance-comparison>`__ - `Conclusions <#conclusions>`__ +.. |image0| image:: https://user-images.githubusercontent.com/4547501/229120774-01f4f972-424d-4280-8395-220dd432985a.png + Prerequisites ------------- -.. |image0| image:: https://user-images.githubusercontent.com/4547501/229120774-01f4f972-424d-4280-8395-220dd432985a.png + .. code:: ipython3 - %pip install -q "openvino>=2023.1.0" seaborn "ultralytics<=8.0.178" onnx + %pip install -q "openvino>=2023.1.0" seaborn "ultralytics<=8.0.178" onnx --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -100,8 +103,10 @@ Prerequisites ) import notebook_utils as utils -Data ----------------------------------------------- +Data +---- + + We will use the same image of the dog sitting on a bicycle for all experiments below. The image is resized and preprocessed to fulfill the @@ -129,19 +134,21 @@ requirements of this particular object detection model. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_4_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_5_0.jpg .. parsed-literal:: - + -Model ------------------------------------------------ +Model +----- + + We decided to go with `YOLOv5n `__, one of the @@ -181,15 +188,17 @@ PyTorch Hub and small enough to see the difference in performance. .. parsed-literal:: Downloading https://github.com/ultralytics/yolov5/releases/download/v7.0/yolov5n.pt to model/yolov5n.pt... - 100%|██████████| 3.87M/3.87M [00:01<00:00, 3.48MB/s] + 100%|██████████| 3.87M/3.87M [00:02<00:00, 1.50MB/s] Fusing layers... YOLOv5n summary: 213 layers, 1867405 parameters, 0 gradients Adding AutoShape... -Hardware --------------------------------------------------- +Hardware +-------- + + The code below lists the available hardware we will use in the benchmarking process. @@ -215,8 +224,10 @@ benchmarking process. CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz -Helper functions ----------------------------------------------------------- +Helper functions +---------------- + + We’re defining a benchmark model function to use for all optimized models below. It runs inference 1000 times, averages the latency time, @@ -350,15 +361,19 @@ the image. utils.show_array(output_img) -Optimizations -------------------------------------------------------- +Optimizations +------------- + + Below, we present the performance tricks for faster inference in the latency mode. We release resources after every benchmarking to be sure the same amount of resource is available for every experiment. -PyTorch model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +PyTorch model +~~~~~~~~~~~~~ + + First, we’re benchmarking the original PyTorch model without any optimizations applied. We will treat it as our baseline. @@ -374,17 +389,19 @@ optimizations applied. We will treat it as our baseline. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_14_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_15_0.jpg .. parsed-literal:: - PyTorch model on CPU. First inference time: 0.0219 seconds - PyTorch model on CPU: 0.0199 seconds per image (50.21 FPS) + PyTorch model on CPU. First inference time: 0.0252 seconds + PyTorch model on CPU: 0.0214 seconds per image (46.73 FPS) + + +ONNX model +~~~~~~~~~~ -ONNX model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The first optimization is exporting the PyTorch model to ONNX and running it in OpenVINO. It’s possible, thanks to the ONNX frontend. It @@ -423,17 +440,19 @@ Representation (IR) to leverage the OpenVINO Runtime. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_17_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_18_0.jpg .. parsed-literal:: - ONNX model on CPU. First inference time: 0.0172 seconds - ONNX model on CPU: 0.0133 seconds per image (75.13 FPS) + ONNX model on CPU. First inference time: 0.0186 seconds + ONNX model on CPU: 0.0124 seconds per image (80.72 FPS) + + +OpenVINO IR model +~~~~~~~~~~~~~~~~~ -OpenVINO IR model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Let’s convert the ONNX model to OpenVINO Intermediate Representation (IR) FP16 and run it. Reducing the precision is one of the well-known @@ -459,17 +478,19 @@ accuracy drop. That’s why we skip that step in this notebook. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_19_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_20_0.jpg .. parsed-literal:: - OpenVINO model on CPU. First inference time: 0.0166 seconds - OpenVINO model on CPU: 0.0133 seconds per image (75.29 FPS) + OpenVINO model on CPU. First inference time: 0.0148 seconds + OpenVINO model on CPU: 0.0123 seconds per image (81.38 FPS) + + +OpenVINO IR model on GPU +~~~~~~~~~~~~~~~~~~~~~~~~ -OpenVINO IR model on GPU -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Usually, a GPU device is faster than a CPU, so let’s run the above model on the GPU. Please note you need to have an Intel GPU and `install @@ -492,8 +513,10 @@ execution. del ov_gpu_model # release resources -OpenVINO IR model + more inference threads -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +OpenVINO IR model + more inference threads +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + There is a possibility to add a config for any device (CPU in this case). We will increase the number of threads to an equal number of our @@ -517,17 +540,19 @@ If it is the case, don’t use it. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_23_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_24_0.jpg .. parsed-literal:: - OpenVINO model + more threads on CPU. First inference time: 0.0156 seconds - OpenVINO model + more threads on CPU: 0.0134 seconds per image (74.72 FPS) + OpenVINO model + more threads on CPU. First inference time: 0.0155 seconds + OpenVINO model + more threads on CPU: 0.0124 seconds per image (80.47 FPS) + + +OpenVINO IR model in latency mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -OpenVINO IR model in latency mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ OpenVINO offers a virtual device called `AUTO `__, @@ -547,17 +572,19 @@ devices as well. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_25_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_26_0.jpg .. parsed-literal:: - OpenVINO model on AUTO. First inference time: 0.0162 seconds - OpenVINO model on AUTO: 0.0136 seconds per image (73.76 FPS) + OpenVINO model on AUTO. First inference time: 0.0156 seconds + OpenVINO model on AUTO: 0.0125 seconds per image (79.73 FPS) + + +OpenVINO IR model in latency mode + shared memory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -OpenVINO IR model in latency mode + shared memory -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ OpenVINO is a C++ toolkit with Python wrappers (API). The default behavior in the Python API is copying the input to the additional buffer @@ -581,27 +608,31 @@ performance! -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_27_0.jpg +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_28_0.jpg .. parsed-literal:: - OpenVINO model + shared memory on AUTO. First inference time: 0.0143 seconds - OpenVINO model + shared memory on AUTO: 0.0054 seconds per image (186.06 FPS) + OpenVINO model + shared memory on AUTO. First inference time: 0.0112 seconds + OpenVINO model + shared memory on AUTO: 0.0054 seconds per image (185.55 FPS) + + +Other tricks +~~~~~~~~~~~~ -Other tricks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There are other tricks for performance improvement, such as quantization and pre-post-processing or dedicated to throughput mode. To get even more from your model, please visit -`111-detection-quantization <111-detection-quantization-with-output.html>`__, -`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__, and -`109-throughput-tricks <109-throughput-tricks-with-output.html>`__. +`111-detection-quantization <../111-detection-quantization>`__, +`118-optimize-preprocessing <../118-optimize-preprocessing>`__, and +`109-throughput-tricks <109-throughput-tricks.ipynb>`__. + +Performance comparison +---------------------- + -Performance comparison ----------------------------------------------------------------- The following graphical comparison is valid for the selected model and hardware simultaneously. If you cannot see any improvement between some @@ -634,11 +665,13 @@ steps, just skip them. -.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png +.. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_31_0.png + + +Conclusions +----------- -Conclusions ------------------------------------------------------ We already showed the steps needed to improve the performance of an object detection model. Even if you experience much better performance diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_14_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_15_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_14_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_15_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_17_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_18_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_17_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_18_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_19_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_20_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_19_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_20_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_23_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_24_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_23_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_24_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_25_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_26_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_25_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_26_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_27_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_28_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_27_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_28_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png deleted file mode 100644 index 6fa3f77dbbf..00000000000 --- a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ff39f8edc5bdcd106ae6c1d49b52edb7342a5a29490b03f3de375b0315155159 -size 57006 diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_31_0.png b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_31_0.png new file mode 100644 index 00000000000..b05a4817148 --- /dev/null +++ b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_31_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:545f14f9a5a612fe9e498660b4640562ad87b9b29df063fa6b354df83c16eaa8 +size 57156 diff --git a/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_4_0.jpg b/docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_5_0.jpg similarity index 100% rename from docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_4_0.jpg rename to docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_5_0.jpg diff --git a/docs/notebooks/109-latency-tricks-with-output_files/index.html b/docs/notebooks/109-latency-tricks-with-output_files/index.html index 75d2c7dfc5e..54ec123587e 100644 --- a/docs/notebooks/109-latency-tricks-with-output_files/index.html +++ b/docs/notebooks/109-latency-tricks-with-output_files/index.html @@ -1,14 +1,14 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/109-latency-tricks-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/109-latency-tricks-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/109-latency-tricks-with-output_files/


../
-109-latency-tricks-with-output_14_0.jpg            31-Oct-2023 00:35              162715
-109-latency-tricks-with-output_17_0.jpg            31-Oct-2023 00:35              162715
-109-latency-tricks-with-output_19_0.jpg            31-Oct-2023 00:35              162715
-109-latency-tricks-with-output_23_0.jpg            31-Oct-2023 00:35              162715
-109-latency-tricks-with-output_25_0.jpg            31-Oct-2023 00:35              162715
-109-latency-tricks-with-output_27_0.jpg            31-Oct-2023 00:35              162715
-109-latency-tricks-with-output_30_0.png            31-Oct-2023 00:35               57006
-109-latency-tricks-with-output_4_0.jpg             31-Oct-2023 00:35              155828
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/109-latency-tricks-with-output_files/


../
+109-latency-tricks-with-output_15_0.jpg            07-Dec-2023 00:49              162715
+109-latency-tricks-with-output_18_0.jpg            07-Dec-2023 00:49              162715
+109-latency-tricks-with-output_20_0.jpg            07-Dec-2023 00:49              162715
+109-latency-tricks-with-output_24_0.jpg            07-Dec-2023 00:49              162715
+109-latency-tricks-with-output_26_0.jpg            07-Dec-2023 00:49              162715
+109-latency-tricks-with-output_28_0.jpg            07-Dec-2023 00:49              162715
+109-latency-tricks-with-output_31_0.png            07-Dec-2023 00:49               57156
+109-latency-tricks-with-output_5_0.jpg             07-Dec-2023 00:49              155828
 

diff --git a/docs/notebooks/109-throughput-tricks-with-output.rst b/docs/notebooks/109-throughput-tricks-with-output.rst index 446d6beac1b..38c1b8cd306 100644 --- a/docs/notebooks/109-throughput-tricks-with-output.rst +++ b/docs/notebooks/109-throughput-tricks-with-output.rst @@ -24,8 +24,8 @@ The quantization and pre-post-processing API are not included here as they change the precision (quantization) or processing graph (prepostprocessor). You can find examples of how to apply them to optimize performance on OpenVINO IR files in -`111-detection-quantization <111-detection-quantization-with-output.html>`__ and -`118-optimize-preprocessing `__. +`111-detection-quantization <../111-detection-quantization>`__ and +`118-optimize-preprocessing <../118-optimize-preprocessing>`__. |image0| @@ -41,11 +41,12 @@ optimize performance on OpenVINO IR files in result in different performance. A similar notebook focused on the latency mode is available -`here <109-latency-tricks-with-output.html>`__. +`here <109-latency-tricks.ipynb>`__. **Table of contents:** +- `Prerequisites <#prerequisites>`__ - `Data <#data>`__ - `Model <#model>`__ - `Hardware <#hardware>`__ @@ -70,14 +71,16 @@ A similar notebook focused on the latency mode is available - `Performance comparison <#performance-comparison>`__ - `Conclusions <#conclusions>`__ +.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/4547501/ac17148c-bee9-43aa-87fc-ead61ac75f1d + Prerequisites ------------- -.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/4547501/ac17148c-bee9-43aa-87fc-ead61ac75f1d + .. code:: ipython3 - %pip install -q "openvino>=2023.1.0" "ultralytics<=8.0.178" seaborn ultralytics onnx + %pip install -q "openvino>=2023.1.0" "ultralytics<=8.0.178" seaborn onnx --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -99,8 +102,10 @@ Prerequisites ) import notebook_utils as utils -Data ----------------------------------------------- +Data +---- + + We will use the same image of the dog sitting on a bicycle copied 1000 times to simulate the video with 1000 frames (about 33s). The image is @@ -134,19 +139,21 @@ object detection model. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_4_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_5_0.jpg .. parsed-literal:: - + -Model ------------------------------------------------ +Model +----- + + We decided to go with `YOLOv5n `__, one of the @@ -186,8 +193,10 @@ PyTorch Hub and small enough to see the difference in performance. requirements: /opt/home/k8sworker/.cache/torch/hub/requirements.txt not found, check failed. -Hardware --------------------------------------------------- +Hardware +-------- + + The code below lists the available hardware we will use in the benchmarking process. @@ -213,8 +222,10 @@ benchmarking process. CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz -Helper functions ----------------------------------------------------------- +Helper functions +---------------- + + We’re defining a benchmark model function to use for all optimizations below. It runs inference for 1000 frames and prints average frames per @@ -353,15 +364,19 @@ the image. utils.show_array(output_img) -Optimizations -------------------------------------------------------- +Optimizations +------------- + + Below, we present the performance tricks for faster inference in the throughput mode. We release resources after every benchmarking to be sure the same amount of resource is available for every experiment. -PyTorch model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +PyTorch model +~~~~~~~~~~~~~ + + First, we’re benchmarking the original PyTorch model without any optimizations applied. We will treat it as our baseline. @@ -377,17 +392,19 @@ optimizations applied. We will treat it as our baseline. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_14_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_15_0.jpg .. parsed-literal:: - PyTorch model on CPU. First inference time: 0.0292 seconds - PyTorch model on CPU: 0.0210 seconds per image (47.67 FPS) + PyTorch model on CPU. First inference time: 0.0220 seconds + PyTorch model on CPU: 0.0208 seconds per image (48.18 FPS) + + +OpenVINO IR model +~~~~~~~~~~~~~~~~~ -OpenVINO IR model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The first optimization is exporting the PyTorch model to OpenVINO Intermediate Representation (IR) FP16 and running it. Reducing the @@ -422,17 +439,19 @@ step in this notebook. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_17_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_18_0.jpg .. parsed-literal:: - OpenVINO model on CPU. First inference time: 0.0182 seconds - OpenVINO model on CPU: 0.0073 seconds per image (136.13 FPS) + OpenVINO model on CPU. First inference time: 0.0156 seconds + OpenVINO model on CPU: 0.0071 seconds per image (141.19 FPS) + + +OpenVINO IR model + bigger batch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -OpenVINO IR model + bigger batch -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Batch processing often gives higher throughput as more inputs are processed at once. To use bigger batches (than 1), we must convert the @@ -478,17 +497,19 @@ hardware and model. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_20_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_21_0.jpg .. parsed-literal:: - OpenVINO model + bigger batch on CPU. First inference time: 0.0502 seconds - OpenVINO model + bigger batch on CPU: 0.0076 seconds per image (131.86 FPS) + OpenVINO model + bigger batch on CPU. First inference time: 0.0481 seconds + OpenVINO model + bigger batch on CPU: 0.0069 seconds per image (145.67 FPS) + + +Asynchronous processing +~~~~~~~~~~~~~~~~~~~~~~~ -Asynchronous processing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Asynchronous mode means that OpenVINO immediately returns from an inference call and doesn’t wait for the result. It requires more @@ -526,8 +547,10 @@ the pipeline. del infer_queue # release resources return fps -OpenVINO IR model in throughput mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +OpenVINO IR model in throughput mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + OpenVINO allows specifying a performance hint changing the internal configuration of the device. There are three different hints: @@ -548,17 +571,19 @@ feature, which sets the batch size to the optimal level. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_24_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_25_0.jpg .. parsed-literal:: - OpenVINO model on CPU (THROUGHPUT). First inference time: 0.0274 seconds - OpenVINO model on CPU (THROUGHPUT): 0.0040 seconds per image (249.34 FPS) + OpenVINO model on CPU (THROUGHPUT). First inference time: 0.0254 seconds + OpenVINO model on CPU (THROUGHPUT): 0.0040 seconds per image (250.82 FPS) + + +OpenVINO IR model in throughput mode on GPU +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -OpenVINO IR model in throughput mode on GPU -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Usually, a GPU device provides more frames per second than a CPU, so let’s run the above model on the GPU. Please note you need to have an @@ -581,8 +606,10 @@ execution. del ov_gpu_model # release resources -OpenVINO IR model in throughput mode on AUTO -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +OpenVINO IR model in throughput mode on AUTO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + OpenVINO offers a virtual device called `AUTO `__, @@ -599,17 +626,19 @@ performance hint. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_28_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_29_0.jpg .. parsed-literal:: - OpenVINO model on AUTO (THROUGHPUT). First inference time: 0.0247 seconds - OpenVINO model on AUTO (THROUGHPUT): 0.0040 seconds per image (248.93 FPS) + OpenVINO model on AUTO (THROUGHPUT). First inference time: 0.0231 seconds + OpenVINO model on AUTO (THROUGHPUT): 0.0040 seconds per image (251.86 FPS) + + +OpenVINO IR model in cumulative throughput mode on AUTO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -OpenVINO IR model in cumulative throughput mode on AUTO -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The AUTO device in throughput mode will select the best, but one physical device to bring the highest throughput. However, if we have @@ -625,29 +654,33 @@ activate all devices. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_30_0.jpg +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_31_0.jpg .. parsed-literal:: - OpenVINO model on AUTO (CUMULATIVE THROUGHPUT). First inference time: 0.0258 seconds - OpenVINO model on AUTO (CUMULATIVE THROUGHPUT): 0.0040 seconds per image (250.04 FPS) + OpenVINO model on AUTO (CUMULATIVE THROUGHPUT). First inference time: 0.0260 seconds + OpenVINO model on AUTO (CUMULATIVE THROUGHPUT): 0.0040 seconds per image (251.22 FPS) + + +Other tricks +~~~~~~~~~~~~ -Other tricks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There are other tricks for performance improvement, such as advanced options, quantization and pre-post-processing or dedicated to latency mode. To get even more from your model, please visit `advanced throughput options `__, -`109-latency-tricks <109-latency-tricks-with-output.html>`__, -`111-detection-quantization <111-detection-quantization-with-output.html>`__, and -`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__. +`109-latency-tricks <109-latency-tricks.ipynb>`__, +`111-detection-quantization <../111-detection-quantization>`__, and +`118-optimize-preprocessing <../118-optimize-preprocessing>`__. + +Performance comparison +---------------------- + -Performance comparison ----------------------------------------------------------------- The following graphical comparison is valid for the selected model and hardware simultaneously. If you cannot see any improvement between some @@ -679,11 +712,13 @@ steps, just skip them. -.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_33_0.png +.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_34_0.png + + +Conclusions +----------- -Conclusions ------------------------------------------------------ We already showed the steps needed to improve the throughput of an object detection model. Even if you experience much better performance diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_14_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_15_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_14_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_15_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_17_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_18_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_17_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_18_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_20_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_21_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_20_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_21_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_24_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_25_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_24_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_25_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_28_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_29_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_28_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_29_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_30_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_31_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_30_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_31_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_33_0.png b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_33_0.png deleted file mode 100644 index 631ed0cf8ec..00000000000 --- a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_33_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:10e27ed40dfd078777a2cd9513a00136b6327571b82ef0af6485b8ea5234dcfa -size 62451 diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_34_0.png b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_34_0.png new file mode 100644 index 00000000000..0696417c908 --- /dev/null +++ b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_34_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94c59d38427a1ec4994131e2bbefc03b0ebdf0fde6f20956a3775faa4e5f451 +size 62516 diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_4_0.jpg b/docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_5_0.jpg similarity index 100% rename from docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_4_0.jpg rename to docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_5_0.jpg diff --git a/docs/notebooks/109-throughput-tricks-with-output_files/index.html b/docs/notebooks/109-throughput-tricks-with-output_files/index.html index 1ec6286d016..33c9880ea6a 100644 --- a/docs/notebooks/109-throughput-tricks-with-output_files/index.html +++ b/docs/notebooks/109-throughput-tricks-with-output_files/index.html @@ -1,14 +1,14 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/109-throughput-tricks-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/109-throughput-tricks-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/109-throughput-tricks-with-output_files/


../
-109-throughput-tricks-with-output_14_0.jpg         31-Oct-2023 00:35              162715
-109-throughput-tricks-with-output_17_0.jpg         31-Oct-2023 00:35              162715
-109-throughput-tricks-with-output_20_0.jpg         31-Oct-2023 00:35              162715
-109-throughput-tricks-with-output_24_0.jpg         31-Oct-2023 00:35              162715
-109-throughput-tricks-with-output_28_0.jpg         31-Oct-2023 00:35              162715
-109-throughput-tricks-with-output_30_0.jpg         31-Oct-2023 00:35              162715
-109-throughput-tricks-with-output_33_0.png         31-Oct-2023 00:35               62451
-109-throughput-tricks-with-output_4_0.jpg          31-Oct-2023 00:35              155828
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/109-throughput-tricks-with-output_files/


../
+109-throughput-tricks-with-output_15_0.jpg         07-Dec-2023 00:49              162715
+109-throughput-tricks-with-output_18_0.jpg         07-Dec-2023 00:49              162715
+109-throughput-tricks-with-output_21_0.jpg         07-Dec-2023 00:49              162715
+109-throughput-tricks-with-output_25_0.jpg         07-Dec-2023 00:49              162715
+109-throughput-tricks-with-output_29_0.jpg         07-Dec-2023 00:49              162715
+109-throughput-tricks-with-output_31_0.jpg         07-Dec-2023 00:49              162715
+109-throughput-tricks-with-output_34_0.png         07-Dec-2023 00:49               62516
+109-throughput-tricks-with-output_5_0.jpg          07-Dec-2023 00:49              155828
 

diff --git a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst index 4ddd722e7a1..f8762186f0c 100644 --- a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst +++ b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst @@ -24,13 +24,13 @@ This third tutorial in the series shows how to: All notebooks in this series: - `Data Preparation for 2D Segmentation of 3D Medical - Data `__ + Data `__ - `Train a 2D-UNet Medical Imaging Model with PyTorch - Lightning `__ + Lightning `__ - Convert and Quantize a Segmentation Model and Show Live Inference (this notebook) - `Live Inference and Benchmark CT-scan - data <110-ct-scan-live-inference-with-output.html>`__ + data <110-ct-scan-live-inference.ipynb>`__ Instructions ------------ @@ -39,7 +39,7 @@ This notebook needs a trained UNet model. We provide a pre-trained model, trained for 20 epochs with the full `Kits-19 `__ frames dataset, which has an F1 score on the validation set of 0.9. The training code is -available in `this notebook `__. +available in `this notebook `__. NNCF for PyTorch models requires a C++ compiler. On Windows, install `Microsoft Visual Studio @@ -66,8 +66,7 @@ purposes, use a representative dataset for quantizing the model. - `Metric <#metric>`__ - `Quantization <#quantization>`__ -- `Compare FP32 and INT8 - Model <#compare-fp-and-int-model>`__ +- `Compare FP32 and INT8 Model <#compare-fp-and-int-model>`__ - `Compare File Size <#compare-file-size>`__ - `Compare Metrics for the original model and the quantized model to @@ -88,7 +87,7 @@ purposes, use a representative dataset for quantizing the model. .. code:: ipython3 - %pip install -q "openvino>=2023.1.0" "monai>=0.9.1,<1.0.0" "torchmetrics>=0.11.0" + %pip install -q "openvino>=2023.1.0" "monai>=0.9.1,<1.0.0" "torchmetrics>=0.11.0" "nncf>=2.6.0" --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -96,8 +95,10 @@ purposes, use a representative dataset for quantizing the model. Note: you may need to restart the kernel to use updated packages. -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -181,10 +182,10 @@ Imports .. parsed-literal:: - 2023-10-30 22:43:08.129843: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-30 22:43:08.164608: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-06 22:47:51.629108: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 22:47:51.662883: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-30 22:43:08.732898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-06 22:47:52.221639: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -192,13 +193,15 @@ Imports INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -Settings --------------------------------------------------- +Settings +-------- + + By default, this notebook will download one CT scan from the KITS19 dataset that will be used for quantization. To use the full dataset, set ``BASEDIR`` to the path of the dataset, as prepared according to the -`Data Preparation `__ notebook. +`Data Preparation `__ notebook. .. code:: ipython3 @@ -208,8 +211,10 @@ dataset that will be used for quantization. To use the full dataset, set MODEL_DIR = Path("model") MODEL_DIR.mkdir(exist_ok=True) -Load PyTorch Model ------------------------------------------------------------- +Load PyTorch Model +------------------ + + Download the pre-trained model weights, load the PyTorch model and the ``state_dict`` that was saved after training. The model used in this @@ -217,7 +222,7 @@ notebook is a `BasicUNet `__ model from `MONAI `__. We provide a pre-trained checkpoint. To see how this model performs, check out the `training -notebook `__. +notebook `__. .. code:: ipython3 @@ -254,8 +259,10 @@ notebook `__. -Download CT-scan Data ---------------------------------------------------------------- +Download CT-scan Data +--------------------- + + .. code:: ipython3 @@ -280,16 +287,20 @@ Download CT-scan Data Data for case_00117 exists -Configuration -------------------------------------------------------- +Configuration +------------- + + + +Dataset +~~~~~~~ + -Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``KitsDataset`` class in the next cell expects images and masks in the *``basedir``* directory, in a folder per patient. It is a simplified version of the Dataset class in the `training -notebook `__. +notebook `__. Images are loaded with MONAI’s `LoadImage `__, @@ -381,8 +392,10 @@ kidney pixels to verify that the annotations look correct: .. image:: 110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png -Metric -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Metric +~~~~~~ + + Define a metric to determine the performance of the model. @@ -416,8 +429,10 @@ library. metric.update(label.flatten(), prediction.flatten()) return metric.compute() -Quantization ------------------------------------------------------- +Quantization +------------ + + Before quantizing the model, we compute the F1 score on the ``FP32`` model, for comparison: @@ -454,7 +469,7 @@ this notebook. [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: @@ -497,6 +512,49 @@ steps: ignored_scope=nncf.IgnoredScope(patterns=[".*LeakyReLU.*"]) ) + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + Export the quantized model to ONNX and then convert it to OpenVINO IR model and save it. @@ -512,11 +570,11 @@ model and save it. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:336: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:333: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_low.item() - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:344: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:341: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_high.item() - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: @@ -527,11 +585,15 @@ than quantization. See the `NNCF documentation `__ in the NNCF repository for more information. -Compare FP32 and INT8 Model ---------------------------------------------------------------------- +Compare FP32 and INT8 Model +--------------------------- + + + +Compare File Size +~~~~~~~~~~~~~~~~~ + -Compare File Size -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 @@ -548,8 +610,10 @@ Compare File Size INT8 model size: 1940.55 KB -Compare Metrics for the original model and the quantized model to be sure that there no degradation. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Compare Metrics for the original model and the quantized model to be sure that there no degradation. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -568,8 +632,10 @@ Compare Metrics for the original model and the quantized model to be sure that t INT8 F1: 0.999 -Compare Performance of the FP32 IR Model and Quantized Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Compare Performance of the FP32 IR Model and Quantized Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + To measure the inference performance of the ``FP32`` and ``INT8`` models, we use `Benchmark @@ -606,18 +672,18 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] CPU - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 34.85 ms + [ INFO ] Read model took 26.10 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,?,?,?] @@ -631,7 +697,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add_425) : f32 / [...] / [?,1,16..,16..] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 76.76 ms + [ INFO ] Compile model took 80.42 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -653,9 +719,9 @@ be run in the notebook with ``! benchmark_app`` or [Step 9/11] Creating infer requests and preparing input tensors [ ERROR ] Input x is dynamic. Provide data shapes! Traceback (most recent call last): - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 485, in main + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 485, in main data_queue = get_input_data(paths_to_input, app_inputs_info) - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data raise Exception(f"Input {info.name} is dynamic. Provide data shapes!") Exception: Input x is dynamic. Provide data shapes! @@ -672,32 +738,32 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] CPU - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 30.95 ms + [ INFO ] Read model took 12.76 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x.1 (node: x.1) : f32 / [...] / [1,1,512,512] [ INFO ] Model outputs: - [ INFO ] 578 (node: 578) : f32 / [...] / [1,1,512,512] + [ INFO ] 571 (node: 571) : f32 / [...] / [1,1,512,512] [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] x.1 (node: x.1) : f32 / [N,C,H,W] / [1,1,512,512] [ INFO ] Model outputs: - [ INFO ] 578 (node: 578) : f32 / [...] / [1,1,512,512] + [ INFO ] 571 (node: 571) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 199.95 ms + [ INFO ] Compile model took 188.00 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: main_graph @@ -721,21 +787,23 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x.1' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 33.43 ms + [ INFO ] First inference took 30.70 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 954 iterations - [ INFO ] Duration: 15006.26 ms + [ INFO ] Count: 971 iterations + [ INFO ] Duration: 15006.86 ms [ INFO ] Latency: - [ INFO ] Median: 15.47 ms - [ INFO ] Average: 15.52 ms - [ INFO ] Min: 15.18 ms - [ INFO ] Max: 19.34 ms - [ INFO ] Throughput: 63.57 FPS + [ INFO ] Median: 15.20 ms + [ INFO ] Average: 15.24 ms + [ INFO ] Min: 14.91 ms + [ INFO ] Max: 16.90 ms + [ INFO ] Throughput: 64.70 FPS + + +Visually Compare Inference Results +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Visually Compare Inference Results -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Visualize the results of the model on four slices of the validation set. Compare the results of the ``FP32`` IR model with the results of the @@ -814,15 +882,17 @@ seed is displayed to enable reproducing specific runs of this cell. .. parsed-literal:: - Visualizing results with seed 1698702266 + Visualizing results with seed 1701899334 .. image:: 110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png -Show Live Inference -------------------------------------------------------------- +Show Live Inference +------------------- + + To show live inference on the model in the notebook, we will use the asynchronous processing feature of OpenVINO. @@ -839,8 +909,10 @@ printed. **NOTE**: If you experience flickering on Firefox, consider using Chrome or Edge to run this notebook. -Load Model and List of Image Files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load Model and List of Image Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + We load the segmentation model to OpenVINO Runtime with ``SegmentationModel``, based on the `Open Model @@ -866,8 +938,10 @@ overlay of the segmentation mask on the original image/frame. case_00117, 69 images -Show Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Inference +~~~~~~~~~~~~~~ + + In the next cell, we run the ``show_live_inference`` function, which loads the ``segmentation_model`` to the specified ``device`` (using @@ -891,24 +965,29 @@ performs inference, and displays the results on the frames loaded in .. parsed-literal:: - Loaded model to CPU in 0.17 seconds. - Total time for 68 frames: 3.43 seconds, fps:20.10 + Loaded model to CPU in 0.18 seconds. + Total time for 68 frames: 2.73 seconds, fps:25.30 -References ----------------------------------------------------- +References +---------- -**OpenVINO** -- `NNCF Repository `__ -- `Neural Network Compression Framework for fast model -inference `__ -- `OpenVINO API Tutorial <002-openvino-api-with-output.html>`__ -- `OpenVINO PyPI (pip install openvino-dev) `__ -**Kits19 Data** +**OpenVINO** - `NNCF +Repository `__ - `Neural +Network Compression Framework for fast model +inference `__ - `OpenVINO API +Tutorial <002-openvino-api-with-output.html>`__ - `OpenVINO +PyPI (pip install +openvino-dev) `__ -- `Kits19 Challenge Homepage `__ -- `Kits19 GitHub Repository `__ -- `The KiTS19 Challenge Data: 300 Kidney Tumor Cases with Clinical Context, CT Semantic Segmentations, and Surgical Outcomes `__ -- `The state of the art in kidney and kidney tumor segmentation in contrast-enhanced CT imaging: Results of the KiTS19 challenge `__ +**Kits19 Data** - `Kits19 Challenge +Homepage `__ - `Kits19 GitHub +Repository `__ - `The KiTS19 +Challenge Data: 300 Kidney Tumor Cases with Clinical Context, CT +Semantic Segmentations, and Surgical +Outcomes `__ - `The state of the art +in kidney and kidney tumor segmentation in contrast-enhanced CT imaging: +Results of the KiTS19 +challenge `__ diff --git a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png index adf48a5b97d..c9b0c7506f3 100644 --- a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png +++ b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eb2418bea000be01c041f45ac09c14c701e09a7b1ffbc551fbc9f7ce4c4fb6fd +oid sha256:8d5031a8fde68c06c9d1585a57d549809af699f8411c9021d3571718ae037908 size 158997 diff --git a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png index 9b09719c5d4..696c1b84369 100644 --- a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png +++ b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6503457981c8d2d24c46d2879b1a499fb4143c566a5196b903b50b49c4094cc2 -size 378309 +oid sha256:04761a5918e007435a0a6d8e1e44df408ba32326558935ebb4c47c48aa1d41cc +size 385394 diff --git a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/index.html b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/index.html index e8d45cd1099..4c06e712a0f 100644 --- a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/index.html +++ b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/index.html @@ -1,9 +1,9 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/


../
-110-ct-segmentation-quantize-nncf-with-output_1..> 31-Oct-2023 00:35              158997
-110-ct-segmentation-quantize-nncf-with-output_3..> 31-Oct-2023 00:35              378309
-110-ct-segmentation-quantize-nncf-with-output_4..> 31-Oct-2023 00:35               73812
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/


../
+110-ct-segmentation-quantize-nncf-with-output_1..> 07-Dec-2023 00:49              158997
+110-ct-segmentation-quantize-nncf-with-output_3..> 07-Dec-2023 00:49              385394
+110-ct-segmentation-quantize-nncf-with-output_4..> 07-Dec-2023 00:49               73812
 

diff --git a/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst b/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst index 8fdf5bea838..31197351669 100644 --- a/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst +++ b/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst @@ -31,8 +31,7 @@ quantization, not demanding the fine-tuning of the model. - `Settings <#settings>`__ - `Download and Prepare Tiny ImageNet dataset <#download-and-prepare-tiny-imagenet-dataset>`__ - - `Helpers classes and - functions <#helpers-classes-and-functions>`__ + - `Helpers classes and functions <#helpers-classes-and-functions>`__ - `Validation function <#validation-function>`__ - `Create and load original uncompressed model <#create-and-load-original-uncompressed-model>`__ @@ -42,8 +41,7 @@ quantization, not demanding the fine-tuning of the model. - `Model quantization and benchmarking <#model-quantization-and-benchmarking>`__ - - `I. Evaluate the loaded - model <#i-evaluate-the-loaded-model>`__ + - `I. Evaluate the loaded model <#i-evaluate-the-loaded-model>`__ - `II. Create and initialize quantization <#ii-create-and-initialize-quantization>`__ - `III. Convert the models to OpenVINO Intermediate Representation @@ -52,8 +50,10 @@ quantization, not demanding the fine-tuning of the model. - `IV. Compare performance of INT8 model and FP32 model in OpenVINO <#iv-compare-performance-of-int-model-and-fp-model-in-openvino>`__ -Preparations ------------------------------------------------------- +Preparations +------------ + + .. code:: ipython3 @@ -105,8 +105,10 @@ Preparations os.environ["LIB"] = os.pathsep.join(b.library_dirs) print(f"Added {vs_dir} to PATH") -Imports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Imports +~~~~~~~ + + .. code:: ipython3 @@ -133,8 +135,10 @@ Imports INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Settings +~~~~~~~~ + + .. code:: ipython3 @@ -174,12 +178,14 @@ Settings .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth') -Download and Prepare Tiny ImageNet dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download and Prepare Tiny ImageNet dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + - 100k images of shape 3x64x64, - 200 different classes: snake, spider, cat, truck, grasshopper, gull, @@ -238,8 +244,10 @@ Download and Prepare Tiny ImageNet dataset Successfully downloaded and extracted dataset to: output -Helpers classes and functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Helpers classes and functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The code below will help to count accuracy and visualize validation process. @@ -304,8 +312,10 @@ process. return res -Validation function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validation function +~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -357,8 +367,10 @@ Validation function ) return top1.avg -Create and load original uncompressed model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create and load original uncompressed model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + ResNet-50 from the `torchivision repository `__ is pre-trained on @@ -385,8 +397,10 @@ values. model = create_model(MODEL_DIR / fp32_checkpoint_filename) -Create train and validation DataLoaders -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create train and validation DataLoaders +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -435,15 +449,19 @@ Create train and validation DataLoaders train_loader, val_loader = create_dataloaders() -Model quantization and benchmarking ------------------------------------------------------------------------------ +Model quantization and benchmarking +----------------------------------- + + With the validation pipeline, model files, and data-loading procedures for model calibration now prepared, it’s time to proceed with the actual post-training quantization using NNCF. -I. Evaluate the loaded model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +I. Evaluate the loaded model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -453,20 +471,22 @@ I. Evaluate the loaded model .. parsed-literal:: - Test: [ 0/79] Time 0.260 (0.260) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19) - Test: [10/79] Time 0.232 (0.238) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50) - Test: [20/79] Time 0.233 (0.237) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35) - Test: [30/79] Time 0.233 (0.236) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33) - Test: [40/79] Time 0.236 (0.235) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51) - Test: [50/79] Time 0.232 (0.235) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42) - Test: [60/79] Time 0.230 (0.235) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79) - Test: [70/79] Time 0.244 (0.235) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33) - * Acc@1 60.740 Acc@5 83.960 Total time: 18.416 + Test: [ 0/79] Time 0.257 (0.257) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19) + Test: [10/79] Time 0.233 (0.231) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50) + Test: [20/79] Time 0.223 (0.231) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35) + Test: [30/79] Time 0.231 (0.231) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33) + Test: [40/79] Time 0.232 (0.234) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51) + Test: [50/79] Time 0.226 (0.233) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42) + Test: [60/79] Time 0.225 (0.233) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79) + Test: [70/79] Time 0.232 (0.234) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33) + * Acc@1 60.740 Acc@5 83.960 Total time: 18.296 Test accuracy of FP32 model: 60.740 -II. Create and initialize quantization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +II. Create and initialize quantization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + NNCF enables post-training quantization by adding the quantization layers into the model graph and then using a subset of the training @@ -502,32 +522,59 @@ Guide
+ + + + +.. raw:: html + +
+    
+ + + .. parsed-literal:: - INFO:nncf:Collecting tensor statistics |█████ | 1 / 3 - INFO:nncf:Collecting tensor statistics |██████████ | 2 / 3 - INFO:nncf:Collecting tensor statistics |████████████████| 3 / 3 INFO:nncf:Compiling and loading torch extension: quantized_functions_cpu... INFO:nncf:Finished loading torch extension: quantized_functions_cpu - INFO:nncf:BatchNorm statistics adaptation |█████ | 1 / 3 - INFO:nncf:BatchNorm statistics adaptation |██████████ | 2 / 3 - INFO:nncf:BatchNorm statistics adaptation |████████████████| 3 / 3 + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ 3. Evaluate the new model on the validation set after initialization of @@ -543,16 +590,16 @@ Guide `__ - `Perform Quantization <#perform-quantization>`__ - - `Create Dataset for - Validation <#create-dataset-for-validation>`__ + - `Create Dataset for Validation <#create-dataset-for-validation>`__ - `Run nncf.quantize for Getting an Optimized Model <#run-nncfquantize-for-getting-an-optimized-model>`__ -- `Serialize an OpenVINO IR - model <#serialize-an-openvino-ir-model>`__ +- `Serialize an OpenVINO IR model <#serialize-an-openvino-ir-model>`__ - `Compare Accuracy of the Original and Quantized Models <#compare-accuracy-of-the-original-and-quantized-models>`__ @@ -66,8 +64,10 @@ This tutorial consists of the following steps: DATA_DIR.mkdir(exist_ok=True) MODEL_DIR.mkdir(exist_ok=True) -Prepare the Model ------------------------------------------------------------ +Prepare the Model +----------------- + + Model preparation stage has the following steps: @@ -91,10 +91,10 @@ Model preparation stage has the following steps: Cloning into 'pytorch-cifar-models'... remote: Enumerating objects: 282, done. remote: Counting objects: 100% (281/281), done. - remote: Compressing objects: 100% (95/95), done. - remote: Total 282 (delta 136), reused 269 (delta 129), pack-reused 1 - Receiving objects: 100% (282/282), 9.22 MiB | 3.32 MiB/s, done. - Resolving deltas: 100% (136/136), done. + remote: Compressing objects: 100% (96/96), done. + remote: Total 282 (delta 135), reused 269 (delta 128), pack-reused 1 + Receiving objects: 100% (282/282), 9.22 MiB | 3.72 MiB/s, done. + Resolving deltas: 100% (135/135), done. .. code:: ipython3 @@ -125,8 +125,10 @@ can be found on this ov.save_model(ov_model, MODEL_DIR / "mobilenet_v2.xml") -Prepare Dataset ---------------------------------------------------------- +Prepare Dataset +--------------- + + We will use `CIFAR10 `__ dataset from @@ -158,7 +160,7 @@ Preprocessing for model obtained from training .. parsed-literal:: - 100%|██████████| 170498071/170498071 [01:12<00:00, 2348008.09it/s] + 100%|██████████| 170498071/170498071 [00:48<00:00, 3527631.36it/s] .. parsed-literal:: @@ -166,8 +168,10 @@ Preprocessing for model obtained from training Extracting data/cifar-10-python.tar.gz to data -Perform Quantization --------------------------------------------------------------- +Perform Quantization +-------------------- + + `NNCF `__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -180,8 +184,10 @@ MobileNetV2. The optimization process contains the following steps: 3. Serialize an OpenVINO IR model, using the ``openvino.save_model`` function. -Create Dataset for Validation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create Dataset for Validation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + NNCF is compatible with ``torch.utils.data.DataLoader`` interface. For performing quantization it should be passed into ``nncf.Dataset`` object @@ -205,8 +211,10 @@ model during quantization, in our case, to pick input tensor from pair INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -Run nncf.quantize for Getting an Optimized Model ------------------------------------------------------------------------------------------- +Run nncf.quantize for Getting an Optimized Model +------------------------------------------------ + + ``nncf.quantize`` function accepts model and prepared quantization dataset for performing basic quantization. Optionally, additional @@ -222,16 +230,58 @@ about supported parameters can be found on this .. parsed-literal:: - 2023-10-30 22:54:06.313060: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-30 22:54:06.344685: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-06 23:00:00.245123: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 23:00:00.276123: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-30 22:54:06.959396: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - Statistics collection: 100%|██████████| 300/300 [00:09<00:00, 31.98it/s] - Applying Fast Bias correction: 100%|██████████| 36/36 [00:01<00:00, 20.03it/s] + 2023-12-06 23:00:00.791927: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + +Serialize an OpenVINO IR model +------------------------------ -Serialize an OpenVINO IR model ------------------------------------------------------------------------- Similar to ``ov.convert_model``, quantized model is ``ov.Model`` object which ready to be loaded into device and can be serialized on disk using @@ -241,8 +291,10 @@ which ready to be loaded into device and can be serialized on disk using ov.save_model(quant_ov_model, MODEL_DIR / "quantized_mobilenet_v2.xml") -Compare Accuracy of the Original and Quantized Models ------------------------------------------------------------------------------------------------ +Compare Accuracy of the Original and Quantized Models +----------------------------------------------------- + + .. code:: ipython3 @@ -259,8 +311,10 @@ Compare Accuracy of the Original and Quantized Models total += 1 return correct / total -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -321,8 +375,10 @@ select device from dropdown list for running inference using OpenVINO Accuracy of the optimized model: 93.54% -Compare Performance of the Original and Quantized Models --------------------------------------------------------------------------------------------------- +Compare Performance of the Original and Quantized Models +-------------------------------------------------------- + + Finally, measure the inference performance of the ``FP32`` and ``INT8`` models, using `Benchmark @@ -348,18 +404,18 @@ Tool -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/113-image-classification-quantization-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/113-image-classification-quantization-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/113-image-classification-quantization-with-output_files/


../
-113-image-classification-quantization-with-outp..> 31-Oct-2023 00:35               14855
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/113-image-classification-quantization-with-output_files/


../
+113-image-classification-quantization-with-outp..> 07-Dec-2023 00:49               14855
 

diff --git a/docs/notebooks/115-async-api-with-output.rst b/docs/notebooks/115-async-api-with-output.rst index 51f74ce38d7..3850ebc5ed1 100644 --- a/docs/notebooks/115-async-api-with-output.rst +++ b/docs/notebooks/115-async-api-with-output.rst @@ -13,6 +13,7 @@ requests) rather than wait for the current inference to complete first. **Table of contents:** + - `Imports <#imports>`__ - `Prepare model and data processing <#prepare-model-and-data-processing>`__ @@ -36,8 +37,7 @@ requests) rather than wait for the current inference to complete first. - `AsyncInferQueue <#asyncinferqueue>`__ - `Setting Callback <#setting-callback>`__ - - `Test the performance with - AsyncInferQueue <#test-the-performance-with-asyncinferqueue>`__ + - `Test the performance with AsyncInferQueue <#test-the-performance-with-asyncinferqueue>`__ Imports ------- @@ -46,8 +46,15 @@ Imports .. code:: ipython3 - # %pip install -q "openvino>=2023.1.0" - # %pip install -q opencv-python matplotlib + %pip install -q "openvino>=2023.1.0" + %pip install -q opencv-python matplotlib + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + .. code:: ipython3 @@ -306,7 +313,7 @@ Test performance in Sync Mode .. parsed-literal:: Source ended - average throuput in sync mode: 38.27 fps + average throuput in sync mode: 40.67 fps Async Mode @@ -445,7 +452,7 @@ Test the performance in Async Mode .. parsed-literal:: Source ended - average throuput in async mode: 72.15 fps + average throuput in async mode: 74.75 fps Compare the performance @@ -588,5 +595,5 @@ Test the performance with ``AsyncInferQueue`` .. parsed-literal:: - average throughput in async mode with async infer queue: 105.36 fps + average throughput in async mode with async infer queue: 111.75 fps diff --git a/docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_21_0.png b/docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_21_0.png index dcc102dd691..c452c1a5664 100644 --- a/docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_21_0.png +++ b/docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_21_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e4f523a824b6e628ef48fa654a0af2dedb2661f23bf18bc31d1e9cc37540fccd -size 30440 +oid sha256:8b8c8b61f0bbb25c280a3e72cf2172fd29bf11668231cb3d2527c1b8a05307f2 +size 30406 diff --git a/docs/notebooks/115-async-api-with-output_files/index.html b/docs/notebooks/115-async-api-with-output_files/index.html index 87539a38a88..376097c1329 100644 --- a/docs/notebooks/115-async-api-with-output_files/index.html +++ b/docs/notebooks/115-async-api-with-output_files/index.html @@ -1,10 +1,10 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/115-async-api-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/115-async-api-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/115-async-api-with-output_files/


../
-115-async-api-with-output_15_0.png                 15-Nov-2023 00:43                4307
-115-async-api-with-output_19_0.png                 15-Nov-2023 00:43                4307
-115-async-api-with-output_21_0.png                 15-Nov-2023 00:43               30440
-115-async-api-with-output_27_0.png                 15-Nov-2023 00:43                4307
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/115-async-api-with-output_files/


../
+115-async-api-with-output_15_0.png                 07-Dec-2023 00:49                4307
+115-async-api-with-output_19_0.png                 07-Dec-2023 00:49                4307
+115-async-api-with-output_21_0.png                 07-Dec-2023 00:49               30406
+115-async-api-with-output_27_0.png                 07-Dec-2023 00:49                4307
 

diff --git a/docs/notebooks/116-sparsity-optimization-with-output.rst b/docs/notebooks/116-sparsity-optimization-with-output.rst index f02e7081021..b6eb46e54c5 100644 --- a/docs/notebooks/116-sparsity-optimization-with-output.rst +++ b/docs/notebooks/116-sparsity-optimization-with-output.rst @@ -37,13 +37,15 @@ consists of the following steps: performance <#benchmark-quantized-sparse-inference-performance>`__ - `When this might be helpful <#when-this-might-be-helpful>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + .. code:: ipython3 %pip install -q "openvino>=2023.1.0" - %pip install -q "git+https://github.com/huggingface/optimum-intel.git" datasets onnx onnxruntime + %pip install -q "git+https://github.com/huggingface/optimum-intel.git" datasets onnx transformers>=4.33.0 --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -52,8 +54,10 @@ Prerequisites Note: you may need to restart the kernel to use updated packages. -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -73,16 +77,16 @@ Imports .. parsed-literal:: No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - 2023-10-30 22:57:12.569340: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-30 22:57:12.603049: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-06 23:02:39.282111: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 23:02:39.316382: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-30 22:57:13.131994: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations - warnings.warn( + 2023-12-06 23:02:40.030243: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + +Download, quantize and sparsify the model, using Hugging Face Optimum API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download, quantize and sparsify the model, using Hugging Face Optimum API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The first step is to download a quantized sparse transformers which has been translated to OpenVINO IR. Then, it will be put through a @@ -113,7 +117,7 @@ model card on Hugging Face. .. parsed-literal:: Compiling the model to CPU ... - Set CACHE_DIR to /opt/home/k8sworker/.cache/huggingface/hub/models--OpenVINO--bert-base-uncased-sst2-int8-unstructured80/snapshots/dc44eb46300882463d50ee847e0f6485bad3cdad/model_cache + Setting OpenVINO CACHE_DIR to /opt/home/k8sworker/.cache/huggingface/hub/models--OpenVINO--bert-base-uncased-sst2-int8-unstructured80/snapshots/dc44eb46300882463d50ee847e0f6485bad3cdad/model_cache .. parsed-literal:: @@ -147,8 +151,10 @@ the IRs into a single folder. -Benchmark quantized dense inference performance ------------------------------------------------------------------------------------------ +Benchmark quantized dense inference performance +----------------------------------------------- + + Benchmark dense inference performance using parallel execution on four CPU cores to simulate a small instance in the cloud infrastructure. @@ -187,18 +193,18 @@ as an example. It is recommended to tune based on your applications. [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] CPU - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 77.13 ms + [ INFO ] Read model took 60.26 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?] @@ -209,7 +215,7 @@ as an example. It is recommended to tune based on your applications. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64] - [ INFO ] Reshape model took 25.98 ms + [ INFO ] Reshape model took 24.75 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,64] @@ -218,7 +224,7 @@ as an example. It is recommended to tune based on your applications. [ INFO ] Model outputs: [ INFO ] logits (node: logits) : f32 / [...] / [1,2] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 970.05 ms + [ INFO ] Compile model took 1092.05 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: torch_jit @@ -246,21 +252,23 @@ as an example. It is recommended to tune based on your applications. [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 28.50 ms + [ INFO ] First inference took 27.56 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 9116 iterations - [ INFO ] Duration: 60051.84 ms + [ INFO ] Count: 8952 iterations + [ INFO ] Duration: 60029.00 ms [ INFO ] Latency: - [ INFO ] Median: 26.14 ms - [ INFO ] Average: 26.19 ms - [ INFO ] Min: 24.91 ms - [ INFO ] Max: 41.99 ms - [ INFO ] Throughput: 151.80 FPS + [ INFO ] Median: 26.46 ms + [ INFO ] Average: 26.52 ms + [ INFO ] Min: 25.37 ms + [ INFO ] Max: 40.49 ms + [ INFO ] Throughput: 149.13 FPS + + +Benchmark quantized sparse inference performance +------------------------------------------------ -Benchmark quantized sparse inference performance ------------------------------------------------------------------------------------------- To enable sparse weight decompression feature, users can add it to runtime config like below. ``CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE`` @@ -300,18 +308,18 @@ for which a layer will be enabled. [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] CPU - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 83.15 ms + [ INFO ] Read model took 61.56 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?] @@ -322,7 +330,7 @@ for which a layer will be enabled. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64] - [ INFO ] Reshape model took 26.29 ms + [ INFO ] Reshape model took 24.68 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,64] @@ -331,7 +339,7 @@ for which a layer will be enabled. [ INFO ] Model outputs: [ INFO ] logits (node: logits) : f32 / [...] / [1,2] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 903.83 ms + [ INFO ] Compile model took 1029.24 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: torch_jit @@ -359,21 +367,23 @@ for which a layer will be enabled. [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 29.52 ms + [ INFO ] First inference took 29.95 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 9128 iterations - [ INFO ] Duration: 60046.44 ms + [ INFO ] Count: 8984 iterations + [ INFO ] Duration: 60026.81 ms [ INFO ] Latency: - [ INFO ] Median: 26.15 ms - [ INFO ] Average: 26.18 ms - [ INFO ] Min: 25.14 ms - [ INFO ] Max: 42.73 ms - [ INFO ] Throughput: 152.02 FPS + [ INFO ] Median: 26.52 ms + [ INFO ] Average: 26.59 ms + [ INFO ] Min: 23.89 ms + [ INFO ] Max: 40.95 ms + [ INFO ] Throughput: 149.67 FPS + + +When this might be helpful +-------------------------- -When this might be helpful --------------------------------------------------------------------- This feature can improve inference performance for models with sparse weights in the scenarios when the model is deployed to handle multiple diff --git a/docs/notebooks/119-tflite-to-openvino-with-output.rst b/docs/notebooks/119-tflite-to-openvino-with-output.rst index 843d31494aa..299489f15e6 100644 --- a/docs/notebooks/119-tflite-to-openvino-with-output.rst +++ b/docs/notebooks/119-tflite-to-openvino-with-output.rst @@ -27,18 +27,21 @@ and do inference with a sample image. Format <#convert-a-model-to-openvino-ir-format>`__ - `Load model using OpenVINO TensorFlow Lite Frontend <#load-model-using-openvino-tensorflow-lite-frontend>`__ -- `Run OpenVINO model - inference <#run-openvino-model-inference>`__ +- `Run OpenVINO model inference <#run-openvino-model-inference>`__ - `Select inference device <#select-inference-device>`__ - `Estimate Model Performance <#estimate-model-performance>`__ -Preparation ------------------------------------------------------ +Preparation +----------- + + + +Install requirements +~~~~~~~~~~~~~~~~~~~~ + -Install requirements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 @@ -59,8 +62,10 @@ Install requirements Note: you may need to restart the kernel to use updated packages. -Imports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Imports +~~~~~~~ + + .. code:: ipython3 @@ -71,8 +76,10 @@ Imports from notebook_utils import download_file, load_image -Download TFLite model ---------------------------------------------------------------- +Download TFLite model +--------------------- + + .. code:: ipython3 @@ -80,7 +87,7 @@ Download TFLite model tflite_model_path = model_dir / "efficientnet_lite0_fp32_2.tflite" ov_model_path = tflite_model_path.with_suffix(".xml") - model_url = "https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/fp32/2?lite-format=tflite" + model_url = "https://www.kaggle.com/models/tensorflow/efficientnet/frameworks/tfLite/variations/lite0-fp32/versions/2?lite-format=tflite" download_file(model_url, tflite_model_path.name, model_dir) @@ -95,12 +102,14 @@ Download TFLite model .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite') -Convert a Model to OpenVINO IR Format -------------------------------------------------------------------------------- +Convert a Model to OpenVINO IR Format +------------------------------------- + + To convert the TFLite model to OpenVINO IR, model conversion Python API can be used. ``ov.convert_model`` function accepts the path to the @@ -127,13 +136,15 @@ For TensorFlow Lite models support, refer to this Model model/efficientnet_lite0_fp32_2.tflite successfully converted and saved to model/efficientnet_lite0_fp32_2.xml -Load model using OpenVINO TensorFlow Lite Frontend --------------------------------------------------------------------------------------------- +Load model using OpenVINO TensorFlow Lite Frontend +-------------------------------------------------- + + TensorFlow Lite models are supported via ``FrontEnd`` API. You may skip conversion to IR and read models directly by OpenVINO runtime API. For more examples supported formats reading via Frontend API, please look -this `tutorial <002-openvino-api-with-output.html>`__. +this `tutorial <../002-openvino-api>`__. .. code:: ipython3 @@ -141,8 +152,10 @@ this `tutorial <002-openvino-api-with-output.html>`__. ov_model = core.read_model(tflite_model_path) -Run OpenVINO model inference ----------------------------------------------------------------------- +Run OpenVINO model inference +---------------------------- + + We can find information about model input preprocessing in its `description `__ @@ -156,8 +169,10 @@ on `TensorFlow Hub `__. resized_image = image.resize((224, 224)) input_tensor = np.expand_dims((np.array(resized_image).astype(np.float32) - 127) / 128, 0) -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -216,8 +231,8 @@ select device from dropdown list for running inference using OpenVINO Predicted label: n02109047 Great Dane with probability 0.715318 -Estimate Model Performance --------------------------------------------------------------------- +Estimate Model Performance +-------------------------- `Benchmark Tool `__ @@ -247,18 +262,18 @@ GPU. [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] CPU - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 29.37 ms + [ INFO ] Read model took 28.35 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: images) : f32 / [...] / [1,224,224,3] @@ -272,7 +287,7 @@ GPU. [ INFO ] Model outputs: [ INFO ] Softmax (node: 63) : f32 / [...] / [1,1000] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 133.03 ms + [ INFO ] Compile model took 147.48 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR @@ -296,15 +311,15 @@ GPU. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 7.30 ms + [ INFO ] First inference took 7.23 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 17562 iterations - [ INFO ] Duration: 15010.30 ms + [ INFO ] Count: 17526 iterations + [ INFO ] Duration: 15005.83 ms [ INFO ] Latency: - [ INFO ] Median: 4.98 ms - [ INFO ] Average: 4.99 ms - [ INFO ] Min: 3.69 ms - [ INFO ] Max: 15.16 ms - [ INFO ] Throughput: 1170.00 FPS + [ INFO ] Median: 5.00 ms + [ INFO ] Average: 5.00 ms + [ INFO ] Min: 2.72 ms + [ INFO ] Max: 15.43 ms + [ INFO ] Throughput: 1167.95 FPS diff --git a/docs/notebooks/119-tflite-to-openvino-with-output_files/index.html b/docs/notebooks/119-tflite-to-openvino-with-output_files/index.html index 783453225f9..ae8ae03dd1a 100644 --- a/docs/notebooks/119-tflite-to-openvino-with-output_files/index.html +++ b/docs/notebooks/119-tflite-to-openvino-with-output_files/index.html @@ -1,8 +1,8 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/119-tflite-to-openvino-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/119-tflite-to-openvino-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/119-tflite-to-openvino-with-output_files/


../
-119-tflite-to-openvino-with-output_16_1.jpg        31-Oct-2023 00:35               68170
-119-tflite-to-openvino-with-output_16_1.png        31-Oct-2023 00:35              621006
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/119-tflite-to-openvino-with-output_files/


../
+119-tflite-to-openvino-with-output_16_1.jpg        07-Dec-2023 00:49               68170
+119-tflite-to-openvino-with-output_16_1.png        07-Dec-2023 00:49              621006
 

diff --git a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst index 8b39f89c50e..034af17d0cb 100644 --- a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst +++ b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst @@ -25,7 +25,6 @@ Runtime `__ - `Imports <#imports>`__ @@ -43,8 +42,10 @@ and do inference with a sample image. - `Async inference pipeline <#async-inference-pipeline>`__ - `Integration preprocessing to model <#integration-preprocessing-to-model>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + Install required packages: @@ -71,8 +72,10 @@ The notebook uses utility functions. The cell below will download the filename="notebook_utils.py", ); -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -90,8 +93,10 @@ Imports # OpenVINO modules import openvino as ov -Settings --------------------------------------------------- +Settings +-------- + + Define model related variables and create corresponding directories: @@ -113,12 +118,14 @@ Define model related variables and create corresponding directories: openvino_ir_path = ir_model_dir / f"{model_name}.xml" - tf_model_url = "https://tfhub.dev/tensorflow/mask_rcnn/inception_resnet_v2_1024x1024/1?tf-hub-format=compressed" + tf_model_url = "https://www.kaggle.com/models/tensorflow/mask-rcnn-inception-resnet-v2/frameworks/tensorFlow2/variations/1024x1024/versions/1?tf-hub-format=compressed" tf_model_archive_filename = f"{model_name}.tar.gz" -Download Model from TensorFlow Hub ----------------------------------------------------------------------------- +Download Model from TensorFlow Hub +---------------------------------- + + Download archive with TensorFlow Instance Segmentation model (`mask_rcnn_inception_resnet_v2_1024x1024 `__) @@ -149,8 +156,10 @@ archive: with tarfile.open(tf_model_dir / tf_model_archive_filename) as file: file.extractall(path=tf_model_dir) -Convert Model to OpenVINO IR ----------------------------------------------------------------------- +Convert Model to OpenVINO IR +---------------------------- + + OpenVINO Model Optimizer Python API can be used to convert the TensorFlow model to OpenVINO IR. @@ -175,11 +184,15 @@ when the model is run in the future. # Save converted OpenVINO IR model to the corresponding directory ov.save_model(ov_model, openvino_ir_path) -Test Inference on the Converted Model -------------------------------------------------------------------------------- +Test Inference on the Converted Model +------------------------------------- + + + +Select inference device +----------------------- + -Select inference device ------------------------------------------------------------------ select device from dropdown list for running inference using OpenVINO @@ -206,31 +219,35 @@ select device from dropdown list for running inference using OpenVINO -Load the Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load the Model +~~~~~~~~~~~~~~ + + .. code:: ipython3 openvino_ir_model = core.read_model(openvino_ir_path) compiled_model = core.compile_model(model=openvino_ir_model, device_name=device.value) -Get Model Information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Get Model Information +~~~~~~~~~~~~~~~~~~~~~ + + Mask R-CNN with Inception ResNet V2 instance segmentation model has one input - a three-channel image of variable size. The input tensor shape is ``[1, height, width, 3]`` with values in ``[0, 255]``. Model output dictionary contains a lot of tensors, we will use only 5 of -them: - -- ``num_detections``: A ``tf.int`` tensor with only one value, the number of detections ``[N]``. -- ``detection_boxes``: A ``tf.float32`` tensor of shape ``[N, 4]`` containing bounding box coordinates in the following order: ``[ymin, xmin, ymax, xmax]``. -- ``detection_classes``: A ``tf.int`` tensor of shape ``[N]`` containing detection class index from the label file. -- ``detection_scores``: A ``tf.float32`` tensor of shape ``[N]`` containing detection scores. -- ``detection_masks``: A ``[batch, max_detections, mask_height, mask_width]`` tensor. - - Note that apixel-wise sigmoid score converter is applied to the detection masks. +them: - ``num_detections``: A ``tf.int`` tensor with only one value, the +number of detections ``[N]``. - ``detection_boxes``: A ``tf.float32`` +tensor of shape ``[N, 4]`` containing bounding box coordinates in the +following order: ``[ymin, xmin, ymax, xmax]``. - ``detection_classes``: +A ``tf.int`` tensor of shape ``[N]`` containing detection class index +from the label file. - ``detection_scores``: A ``tf.float32`` tensor of +shape ``[N]`` containing detection scores. - ``detection_masks``: A +``[batch, max_detections, mask_height, mask_width]`` tensor. Note that a +pixel-wise sigmoid score converter is applied to the detection masks. For more information about model inputs, outputs and their formats, see the `model overview page on TensorFlow @@ -290,8 +307,10 @@ the first (and highest) detection score. -Get an Image for Test Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Get an Image for Test Inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Load and save an image: @@ -336,7 +355,7 @@ Read the image, resize and convert it to the input shape of the network: .. parsed-literal:: - + @@ -344,8 +363,10 @@ Read the image, resize and convert it to the input shape of the network: .. image:: 120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_25_1.png -Perform Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Perform Inference +~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -391,8 +412,10 @@ be extracted from the result. For further model result visualization image_detections_num: [100.] -Inference Result Visualization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Inference Result Visualization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Define utility functions to visualize the inference results @@ -631,24 +654,29 @@ original test image: .. image:: 120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_39_0.png -Next Steps ----------------------------------------------------- +Next Steps +---------- + + This section contains suggestions on how to additionally improve the performance of your application using OpenVINO. -Async inference pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Async inference pipeline +~~~~~~~~~~~~~~~~~~~~~~~~ -The key advantage of the Async API is that when a device is busy with -inference, the application can perform other tasks in parallel (for -example, populating inputs or scheduling other requests) rather than -wait for the current inference to complete first. To understand how to -perform async inference using openvino, refer to the `Async API +The key advantage of the Async +API is that when a device is busy with inference, the application can +perform other tasks in parallel (for example, populating inputs or +scheduling other requests) rather than wait for the current inference to +complete first. To understand how to perform async inference using +openvino, refer to the `Async API tutorial <115-async-api-with-output.html>`__. -Integration preprocessing to model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Integration preprocessing to model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Preprocessing API enables making preprocessing a part of the model reducing application code and dependency on additional image processing diff --git a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_25_1.png b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_25_1.png index 0a3d192dfb9..d28e1510584 100644 --- a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_25_1.png +++ b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_25_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:899a1126af7e881b5b8ad2182133a5334f3c98031a9d1b5d9285a76b44a162fc +oid sha256:62971c6546b399ce0eecbdc81a5cb327c4bebdfa1a6db658e364870db606d761 size 395346 diff --git a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_39_0.png b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_39_0.png index b9851ff3364..99694eb8f03 100644 --- a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_39_0.png +++ b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_39_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d8537cac1002162c69c666fe659effaf3bb23d10d3da276349abe7af2469499 -size 394617 +oid sha256:1359365695e66164137690b56f261eae3364dff68f3e155a3e6b878ff17cea3f +size 393190 diff --git a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/index.html b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/index.html index f0b5ea94559..9911d8830b7 100644 --- a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/index.html +++ b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/index.html @@ -1,8 +1,8 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/120-tensorflow-instance-segmentation-to-openvino-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/120-tensorflow-instance-segmentation-to-openvino-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/120-tensorflow-instance-segmentation-to-openvino-with-output_files/


../
-120-tensorflow-instance-segmentation-to-openvin..> 31-Oct-2023 00:35              395346
-120-tensorflow-instance-segmentation-to-openvin..> 31-Oct-2023 00:35              394617
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/120-tensorflow-instance-segmentation-to-openvino-with-output_files/


../
+120-tensorflow-instance-segmentation-to-openvin..> 07-Dec-2023 00:49              395346
+120-tensorflow-instance-segmentation-to-openvin..> 07-Dec-2023 00:49              393190
 

diff --git a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst index 393738a1168..0a69d8cc2ff 100644 --- a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst +++ b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst @@ -31,8 +31,7 @@ and do inference with a sample image. - `Settings <#settings>`__ - `Download Model from TensorFlow Hub <#download-model-from-tensorflow-hub>`__ -- `Convert Model to OpenVINO - IR <#convert-model-to-openvino-ir>`__ +- `Convert Model to OpenVINO IR <#convert-model-to-openvino-ir>`__ - `Test Inference on the Converted Model <#test-inference-on-the-converted-model>`__ - `Select inference device <#select-inference-device>`__ @@ -51,8 +50,10 @@ and do inference with a sample image. - `Integration preprocessing to model <#integration-preprocessing-to-model>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + Install required packages: @@ -79,8 +80,10 @@ The notebook uses utility functions. The cell below will download the filename="notebook_utils.py", ); -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -97,8 +100,10 @@ Imports # Notebook utils module from notebook_utils import download_file -Settings --------------------------------------------------- +Settings +-------- + + Define model related variables and create corresponding directories: @@ -120,12 +125,14 @@ Define model related variables and create corresponding directories: openvino_ir_path = ir_model_dir / f"{model_name}.xml" - tf_model_url = "https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1?tf-hub-format=compressed" + tf_model_url = "https://www.kaggle.com/models/tensorflow/faster-rcnn-resnet-v1/frameworks/tensorFlow2/variations/faster-rcnn-resnet50-v1-640x640/versions/1?tf-hub-format=compressed" tf_model_archive_filename = f"{model_name}.tar.gz" -Download Model from TensorFlow Hub ----------------------------------------------------------------------------- +Download Model from TensorFlow Hub +---------------------------------- + + Download archive with TensorFlow Object Detection model (`faster_rcnn_resnet50_v1_640x640 `__) @@ -150,7 +157,7 @@ from TensorFlow Hub: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz') @@ -163,8 +170,10 @@ Extract TensorFlow Object Detection model from the downloaded archive: with tarfile.open(tf_model_dir / tf_model_archive_filename) as file: file.extractall(path=tf_model_dir) -Convert Model to OpenVINO IR ----------------------------------------------------------------------- +Convert Model to OpenVINO IR +---------------------------- + + OpenVINO Model Converter Python API can be used to convert the TensorFlow model to OpenVINO IR. @@ -180,7 +189,7 @@ or saved on disk using the ``save_model`` function to reduce loading time when the model is run in the future. See the `Model Converter Developer -Guide `__ +Guide `__ for more information about Model Converter and TensorFlow `models support `__. @@ -191,11 +200,15 @@ support -Get an Image for Test Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Get an Image for Test Inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Load and save an image: @@ -324,7 +343,7 @@ Load and save an image: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_bike.jpg') @@ -352,7 +371,7 @@ Read the image, resize and convert it to the input shape of the network: .. parsed-literal:: - + @@ -360,8 +379,10 @@ Read the image, resize and convert it to the input shape of the network: .. image:: 120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png -Perform Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Perform Inference +~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -471,8 +492,10 @@ outputs will be used. image_detections_num: [300.] -Inference Result Visualization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Inference Result Visualization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Define utility functions to visualize the inference results @@ -612,7 +635,7 @@ Zoo `__: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_91cl.txt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_91cl.txt') @@ -651,24 +674,29 @@ original test image: .. image:: 120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png -Next Steps ----------------------------------------------------- +Next Steps +---------- + + This section contains suggestions on how to additionally improve the performance of your application using OpenVINO. -Async inference pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Async inference pipeline +~~~~~~~~~~~~~~~~~~~~~~~~ -The key advantage of the Async API is that when a device is busy with -inference, the application can perform other tasks in parallel (for -example, populating inputs or scheduling other requests) rather than -wait for the current inference to complete first. To understand how to -perform async inference using openvino, refer to the `Async API +The key advantage of the Async +API is that when a device is busy with inference, the application can +perform other tasks in parallel (for example, populating inputs or +scheduling other requests) rather than wait for the current inference to +complete first. To understand how to perform async inference using +openvino, refer to the `Async API tutorial <115-async-api-with-output.html>`__. -Integration preprocessing to model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Integration preprocessing to model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Preprocessing API enables making preprocessing a part of the model reducing application code and dependency on additional image processing diff --git a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png index 0a3d192dfb9..d28e1510584 100644 --- a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png +++ b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:899a1126af7e881b5b8ad2182133a5334f3c98031a9d1b5d9285a76b44a162fc +oid sha256:62971c6546b399ce0eecbdc81a5cb327c4bebdfa1a6db658e364870db606d761 size 395346 diff --git a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png index 33a6b939145..c767e2ad495 100644 --- a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png +++ b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df9e84273cf7c5c19bf1c4ea8d6b61372f59b04228a95bacd2fac1b97dde6d4a -size 392067 +oid sha256:ec79c39b72b7a139eca9f5b7081efec33a47fa8c55c9a459340774a045f2525b +size 391797 diff --git a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/index.html b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/index.html index 1faa6594c62..e489a487635 100644 --- a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/index.html +++ b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/index.html @@ -1,8 +1,8 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/120-tensorflow-object-detection-to-openvino-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/120-tensorflow-object-detection-to-openvino-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/120-tensorflow-object-detection-to-openvino-with-output_files/


../
-120-tensorflow-object-detection-to-openvino-wit..> 31-Oct-2023 00:35              395346
-120-tensorflow-object-detection-to-openvino-wit..> 31-Oct-2023 00:35              392067
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/120-tensorflow-object-detection-to-openvino-with-output_files/


../
+120-tensorflow-object-detection-to-openvino-wit..> 07-Dec-2023 00:49              395346
+120-tensorflow-object-detection-to-openvino-wit..> 07-Dec-2023 00:49              391797
 

diff --git a/docs/notebooks/121-convert-to-openvino-with-output.rst b/docs/notebooks/121-convert-to-openvino-with-output.rst index e0f554470d2..20b8583863d 100644 --- a/docs/notebooks/121-convert-to-openvino-with-output.rst +++ b/docs/notebooks/121-convert-to-openvino-with-output.rst @@ -2,7 +2,10 @@ OpenVINO™ model conversion API ============================== This notebook shows how to convert a model from original framework -format to OpenVINO Intermediate Representation (IR). Contents: +format to OpenVINO Intermediate Representation (IR). + +**Table of contents:** + - `OpenVINO IR format <#openvino-ir-format>`__ - `IR preparation with Python conversion API and Model Optimizer @@ -38,13 +41,15 @@ format to OpenVINO Intermediate Representation (IR). Contents: .. parsed-literal:: ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - tensorflow 2.13.1 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 3.20.2 which is incompatible. + tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 3.20.2 which is incompatible. Note: you may need to restart the kernel to use updated packages. OpenVINO IR format ------------------ + + OpenVINO `Intermediate Representation (IR) `__ is the proprietary model format of OpenVINO. It is produced after converting a @@ -58,6 +63,8 @@ an ``.xml`` file, containing information about network topology, and a IR preparation with Python conversion API and Model Optimizer command-line tool ------------------------------------------------------------------------------- + + There are two ways to convert a model from the original framework format to OpenVINO IR: Python conversion API and Model Optimizer command-line tool. You can choose one of them based on whichever is most convenient @@ -113,7 +120,7 @@ documentation. conversion into IR. The legacy Frontend is Python based and is available for TensorFlow*, ONNX*, MXNet*, Caffe*, and Kaldi* models. - --input_model INPUT_MODEL, -w INPUT_MODEL, -m INPUT_MODEL + --input_model INPUT_MODEL, -m INPUT_MODEL, -w INPUT_MODEL Tensorflow*: a file with a pre-trained model (binary or text .pb file after freezing). Caffe*: a model proto file with model weights. @@ -678,6 +685,8 @@ documentation. Fetching example models ----------------------- + + This notebook uses two models for conversion examples: - `Distilbert `__ @@ -733,11 +742,11 @@ NLP model from Hugging Face and export it in ONNX format: .. parsed-literal:: - 2023-10-30 23:03:34.054449: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-30 23:03:34.088016: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-06 23:09:08.345195: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 23:09:08.379671: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-30 23:03:34.718197: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:223: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + 2023-12-06 23:09:09.007347: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:223: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mask, torch.tensor(torch.finfo(scores.dtype).min) @@ -976,6 +985,8 @@ Convert PyTorch model to ONNX format: Basic conversion ---------------- + + To convert a model to OpenVINO IR, use the following command: .. code:: ipython3 @@ -1000,8 +1011,8 @@ To convert a model to OpenVINO IR, use the following command: [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html [ SUCCESS ] Generated IR version 11 model. - [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/121-convert-to-openvino/model/distilbert.xml - [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/121-convert-to-openvino/model/distilbert.bin + [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/121-convert-to-openvino/model/distilbert.xml + [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/121-convert-to-openvino/model/distilbert.bin .. code:: ipython3 @@ -1029,6 +1040,8 @@ To convert a model to OpenVINO IR, use the following command: Model conversion parameters --------------------------- + + Both Python conversion API and Model Optimizer command-line tool provide the following capabilities: \* overriding original input shapes for model conversion with ``input`` and ``input_shape`` parameters. `Setting @@ -1055,6 +1068,8 @@ mentioned above to override input shapes and cut the model. Setting Input Shapes ~~~~~~~~~~~~~~~~~~~~ + + Model conversion is supported for models with dynamic input shapes that contain undefined dimensions. However, if the shape of data is not going to change from one inference request to another, it is recommended to @@ -1092,8 +1107,8 @@ guide `__ -- `Get Pytorch model and OpenVINO IR model <#>`__ -- `Define validator and data loader <#>`__ -- `Prepare calibration and validation datasets <#>`__ -- `Prepare validation function <#>`__ -- `Run quantization with accuracy control <#>`__ -- `Compare Performance of the Original and Quantized Models <#>`__ + +- `Prerequisites <#prerequisites>`__ +- `Get Pytorch model and OpenVINO IR + model <#get-pytorch-model-and-openvino-ir-model>`__ + + - `Define validator and data + loader <#define-validator-and-data-loader>`__ + - `Prepare calibration and validation + datasets <#prepare-calibration-and-validation-datasets>`__ + - `Prepare validation function <#prepare-validation-function>`__ + +- `Run quantization with accuracy + control <#run-quantization-with-accuracy-control>`__ +- `Compare Accuracy and Performance of the Original and Quantized + Models <#compare-accuracy-and-performance-of-the-original-and-quantized-models>`__ Prerequisites ----------------------------------- +^^^^^^^^^^^^^ + + Install necessary packages. .. code:: ipython3 %pip install -q "openvino>=2023.1.0" - %pip install "nncf>=2.6.0" - %pip install -q "ultralytics==8.0.43" + %pip install -q "nncf>=2.6.0" + %pip install -q "ultralytics==8.0.43" --extra-index-url https://download.pytorch.org/whl/cpu + +Get Pytorch model and OpenVINO IR model +--------------------------------------- + -Get Pytorch model and OpenVINO IR Model ---------------------------------------------------- Generally, PyTorch models represent an instance of the `torch.nn.Module `__ @@ -115,18 +127,19 @@ Load model. ov_model = ov.Core().read_model(model_path) Define validator and data loader ---------------------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The original model -repository uses a ``Validator`` wrapper, which represents the accuracy -validation pipeline. It creates dataloader and evaluation metrics and -updates metrics on each data batch produced by the dataloader. Besides -that, it is responsible for data preprocessing and results -postprocessing. For class initialization, the configuration should be -provided. We will use the default setup, but it can be replaced with -some parameters overriding to test on custom data. The model has -connected the ``ValidatorClass`` method, which creates a validator class -instance. + + +The original model repository uses a ``Validator`` wrapper, which +represents the accuracy validation pipeline. It creates dataloader and +evaluation metrics and updates metrics on each data batch produced by +the dataloader. Besides that, it is responsible for data preprocessing +and results postprocessing. For class initialization, the configuration +should be provided. We will use the default setup, but it can be +replaced with some parameters overriding to test on custom data. The +model has connected the ``ValidatorClass`` method, which creates a +validator class instance. .. code:: ipython3 @@ -150,7 +163,9 @@ instance. Prepare calibration and validation datasets ---------------------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + We can use one dataset as calibration and validation datasets. Name it ``quantization_dataset``. @@ -176,7 +191,9 @@ We can use one dataset as calibration and validation datasets. Name it Prepare validation function ----------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + .. code:: ipython3 @@ -230,24 +247,22 @@ Prepare validation function validation_fn = partial(validation_ac, validator=validator, log=False) Run quantization with accuracy control ---------------------------------------------------- +-------------------------------------- -You should provide -the calibration dataset and the validation dataset. It can be the same -dataset. -- parameter ``max_drop`` defines the accuracy drop threshold. - The quantization process stops when the degradation of accuracy metric - on the validation dataset is less than the ``max_drop``. The default - value is 0.01. NNCF will stop the quantization and report an error if - the ``max_drop`` value can’t be reached. -- ``drop_type`` defines how the - accuracy drop will be calculated: ABSOLUTE (used by default) or - RELATIVE. -- ``ranking_subset_size`` - size of a subset that is used to - rank layers by their contribution to the accuracy drop. Default value is - 300, and the more samples it has the better ranking, potentially. Here - we use the value 25 to speed up the execution. + +You should provide the calibration dataset and the validation dataset. +It can be the same dataset. - parameter ``max_drop`` defines the +accuracy drop threshold. The quantization process stops when the +degradation of accuracy metric on the validation dataset is less than +the ``max_drop``. The default value is 0.01. NNCF will stop the +quantization and report an error if the ``max_drop`` value can’t be +reached. - ``drop_type`` defines how the accuracy drop will be +calculated: ABSOLUTE (used by default) or RELATIVE. - +``ranking_subset_size`` - size of a subset that is used to rank layers +by their contribution to the accuracy drop. Default value is 300, and +the more samples it has the better ranking, potentially. Here we use the +value 25 to speed up the execution. **NOTE**: Execution can take tens of minutes and requires up to 15 GB of free memory @@ -321,9 +336,11 @@ dataset. Compare Accuracy and Performance of the Original and Quantized Models --------------------------------------------------------------------- -Now we can compare metrics of the Original non-quantized -OpenVINO IR model and Quantized OpenVINO IR model to make sure that the -``max_drop`` is not exceeded. + + +Now we can compare metrics of the Original non-quantized OpenVINO IR +model and Quantized OpenVINO IR model to make sure that the ``max_drop`` +is not exceeded. .. code:: ipython3 diff --git a/docs/notebooks/123-detectron2-to-openvino-with-output.rst b/docs/notebooks/123-detectron2-to-openvino-with-output.rst index 03019a26cbc..373adb96367 100644 --- a/docs/notebooks/123-detectron2-to-openvino-with-output.rst +++ b/docs/notebooks/123-detectron2-to-openvino-with-output.rst @@ -1,5 +1,4 @@ -Convert Detectron2 Models to OpenVINO™ -======================================== +# Convert Detectron2 Models to OpenVINO™ `Detectron2 `__ is Facebook AI Research’s library that provides state-of-the-art detection @@ -16,31 +15,44 @@ using OpenVINO™. We will use ``Faster R-CNN FPN x1`` model and detection and instance segmentation respectively. **Table of contents:** --- -- `Prerequisites <#prerequisites>`__ -- `Define helpers for PyTorch model initialization and conversion <#define-helpers-for-pytorch-model-initialization-and-conversion>`__ -- `Prepare input data <#prepare-input-data>`__ -- `Object Detection <#object-detection>`__ -- `Download PyTorch Detection model <#download-pytorch-detection-model>`__ -- `Convert Detection Model to OpenVINO Intermediate Representation <#convert-detection-model-to-openvino-intermediate-representation>`__ -- `Select inference device <#select-inference-device>`__ -- `Run Detection model inference <#run-detection-model-inference>`__ -- `Instance Segmentation <#instance-segmentation>`__ -- `Download PyTorch Instance Segmentation model <#download-pytorch-instance-segmentation-model>`__ -- `Convert Instance Segmentation Model to OpenVINO Intermediate Representation <#convert-instance-segmentation-model-to-openvino-intermediate-representation>`__ -- `Select inference device <#select-inference-device>`__ -- `Run Instance Segmentation model inference <#run-instance-segmentation-model-inference>`__ + +- `Prerequisites <#prerequisites>`__ + + - `Define helpers for PyTorch model initialization and + conversion <#define-helpers-for-pytorch-model-initialization-and-conversion>`__ + - `Prepare input data <#prepare-input-data>`__ + +- `Object Detection <#object-detection>`__ + + - `Download PyTorch Detection + model <#download-pytorch-detection-model>`__ + - `Convert Detection Model to OpenVINO Intermediate + Representation <#convert-detection-model-to-openvino-intermediate-representation>`__ + - `Select inference device <#select-inference-device>`__ + - `Run Detection model inference <#run-detection-model-inference>`__ + +- `Instance Segmentation <#instance-segmentation>`__ + + - `Download Instance Segmentation PyTorch + model <#download-instance-segmentation-pytorch-model>`__ + - `Convert Instance Segmentation Model to OpenVINO Intermediate + Representation <#convert-instance-segmentation-model-to-openvino-intermediate-representation>`__ + - `Select inference device <#select-inference-device>`__ + - `Run Instance Segmentation model + inference <#run-instance-segmentation-model-inference>`__ Prerequisites -------------------------------------------------------- +------------- + + Install required packages for running model .. code:: ipython3 %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision - %pip install -q "git+https://github.com/facebookresearch/detectron2.git" + %pip install -q "git+https://github.com/facebookresearch/detectron2.git" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "openvino>=2023.1.0" @@ -51,8 +63,10 @@ Install required packages for running model Note: you may need to restart the kernel to use updated packages. -Define helpers for PyTorch model initialization and conversion -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Define helpers for PyTorch model initialization and conversion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Detectron2 provides universal and configurable API for working with models, it means that all steps required for model creation, conversion @@ -139,8 +153,10 @@ simplify model’s structure making it more export-friendly. ov_model = ov.convert_model(traceable_model, example_input=sample_input[0]["image"]) return ov_model -Prepare input data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare input data +~~~~~~~~~~~~~~~~~~ + + For running model conversion and inference we need to provide example input. The cells below download sample image and apply preprocessing @@ -201,11 +217,15 @@ steps based on model specific transformations defined in model config. Now, when all components required for model conversion are prepared, we can consider how to use them on specific examples. -Object Detection ----------------------------------------------------------- +Object Detection +---------------- + + + +Download PyTorch Detection model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + -Download PyTorch Detection model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Download faster_rcnn_R_50_FPN_1x from Detectron Model Zoo. @@ -215,8 +235,10 @@ Download faster_rcnn_R_50_FPN_1x from Detectron Model Zoo. model, cfg = get_model_and_config(model_name) sample_input = get_sample_inputs(image_file, cfg) -Convert Detection Model to OpenVINO Intermediate Representation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert Detection Model to OpenVINO Intermediate Representation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Convert model using ``convert_detectron2_model`` function and ``sample_input`` prepared above. After conversion, model saved on disk @@ -232,8 +254,10 @@ directory. else: ov_model = model_xml_path -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -264,6 +288,8 @@ select device from dropdown list for running inference using OpenVINO Run Detection model inference ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Load our converted model on selected device and run inference on sample input. @@ -362,6 +388,8 @@ provide helpers for wrapping output in original Detectron2 format. Instance Segmentation --------------------- + + As it was discussed above, Detectron2 provides generic approach for working with models for different use cases. The steps that required to convert and run models pretrained for Instance Segmentation use case @@ -370,6 +398,8 @@ will be very similar to Object Detection. Download Instance Segmentation PyTorch model ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 model_name = "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x" @@ -379,6 +409,8 @@ Download Instance Segmentation PyTorch model Convert Instance Segmentation Model to OpenVINO Intermediate Representation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 model_xml_path = MODEL_DIR / (model_name.split("/")[-1] + '.xml') @@ -389,8 +421,10 @@ Convert Instance Segmentation Model to OpenVINO Intermediate Representation else: ov_model = model_xml_path -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -410,6 +444,8 @@ select device from dropdown list for running inference using OpenVINO Run Instance Segmentation model inference ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + In comparison with Object Detection, Instance Segmentation models have additional output that represents instance masks for each object. Our postprocessing function handle this difference. diff --git a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.jpg b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.jpg index 21179d56bf5..cd651b0e2b6 100644 --- a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.jpg +++ b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:856bd51bd1a5dd45f52f6f0f939390f11c5c35b4af4781679a617cb9fe451a41 -size 57591 +oid sha256:d6f08e4b663ac8ea6d0970804111a5ffc4e7aac44f24b2a7121f15c9900fa535 +size 59666 diff --git a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.png b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.png index 22b92c9c1d9..6b0bc4fdbc3 100644 --- a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.png +++ b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f3846096706af5b748d9e673831136adb1a5d912a91783b6c3b700edbd0d8359 -size 508592 +oid sha256:a4a06ea24c3a51aead9c90815b867910e7be07e8b8d443c89ad4003695dcf48d +size 508841 diff --git a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.jpg b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.jpg index f589819418c..cbce435a317 100644 --- a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.jpg +++ b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8668a57174bdee831756b00f1cc479c75ad4515151d4cbf5adfa8a87be595d77 -size 53100 +oid sha256:cacd033467ea9cdf6ea9db93bb70af3837dd492a870bac492683badc7643ed6d +size 55922 diff --git a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.png b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.png index 9516a78e6dc..a5e22ebe6d1 100644 --- a/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.png +++ b/docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2913494596627acf2e77e9846889891f2dd348d6162dfdfade8ec0857698361 -size 456596 +oid sha256:0f1347884309a3631eb6eb2c63a44fa995c158b1e747c5230974eaada4fda4d0 +size 459385 diff --git a/docs/notebooks/123-detectron2-to-openvino-with-output_files/index.html b/docs/notebooks/123-detectron2-to-openvino-with-output_files/index.html index 150ba32dd39..6ddb39d37c3 100644 --- a/docs/notebooks/123-detectron2-to-openvino-with-output_files/index.html +++ b/docs/notebooks/123-detectron2-to-openvino-with-output_files/index.html @@ -1,12 +1,12 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/123-detectron2-to-openvino-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/123-detectron2-to-openvino-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/123-detectron2-to-openvino-with-output_files/


../
-123-detectron2-to-openvino-with-output_22_0.jpg    31-Oct-2023 00:35               57591
-123-detectron2-to-openvino-with-output_22_0.png    31-Oct-2023 00:35              508592
-123-detectron2-to-openvino-with-output_32_0.jpg    31-Oct-2023 00:35               53100
-123-detectron2-to-openvino-with-output_32_0.png    31-Oct-2023 00:35              456596
-123-detectron2-to-openvino-with-output_8_0.jpg     31-Oct-2023 00:35               46858
-123-detectron2-to-openvino-with-output_8_0.png     31-Oct-2023 00:35              503218
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/123-detectron2-to-openvino-with-output_files/


../
+123-detectron2-to-openvino-with-output_22_0.jpg    07-Dec-2023 00:49               59666
+123-detectron2-to-openvino-with-output_22_0.png    07-Dec-2023 00:49              508841
+123-detectron2-to-openvino-with-output_32_0.jpg    07-Dec-2023 00:49               55922
+123-detectron2-to-openvino-with-output_32_0.png    07-Dec-2023 00:49              459385
+123-detectron2-to-openvino-with-output_8_0.jpg     07-Dec-2023 00:49               46858
+123-detectron2-to-openvino-with-output_8_0.png     07-Dec-2023 00:49              503218
 

diff --git a/docs/notebooks/124-hugging-face-hub-with-output.rst b/docs/notebooks/124-hugging-face-hub-with-output.rst index ca4d046f964..863d6facd0c 100644 --- a/docs/notebooks/124-hugging-face-hub-with-output.rst +++ b/docs/notebooks/124-hugging-face-hub-with-output.rst @@ -13,13 +13,13 @@ models, namely |image0| -Throughout this notebook we will learn: - -1. How to load a HF pipeline using the ``transformers`` package and then convert it to OpenVINO. -2. How to load the same pipeline using Optimum Intel package. +Throughout this notebook we will learn: 1. How to load a HF pipeline +using the ``transformers`` package and then convert it to OpenVINO. 2. +How to load the same pipeline using Optimum Intel package. **Table of contents:** + - `Converting a Model from the HF Transformers Package <#converting-a-model-from-the-hf-transformers-package>`__ @@ -97,7 +97,8 @@ Initializing a Model Using the HF Transformers Package -We will use `roberta text sentiment classification `__ +We will use `roberta text sentiment +classification `__ model in our example, it is a transformer-based encoder model pretrained in a special way, please refer to the model card to learn more. @@ -120,7 +121,7 @@ tutorials `__ +We use the OpenVINO `Model +conversion +API `__ to convert the model (this one is implemented in PyTorch) to OpenVINO Intermediate Representation (IR). @@ -235,11 +238,13 @@ original model. This is a rather simple example as the pipeline includes just one encoder model. Contemporary state of the art pipelines often consist of -several model, feel free to explore other OpenVINO tutorials: - -1. `Stable Diffusion v2 `__ -2. `Zero-shot Image Classification with OpenAI CLIP `__ -3. `Controllable Music Generation with MusicGen `__ +several model, feel free to explore other OpenVINO tutorials: 1. `Stable +Diffusion +v2 `__ +2. `Zero-shot Image Classification with OpenAI +CLIP `__ +3. `Controllable Music Generation with +MusicGen `__ The workflow for the ``diffusers`` package is exactly the same. The first example in the list above relies on the ``diffusers``. @@ -292,7 +297,8 @@ the full list of supported devices). For that, just replace the ``AutoModelForXxx`` class with the corresponding ``OVModelForXxx`` class. -You can find more information in `Optimum Intel documentation `__. +You can find more information in `Optimum Intel +documentation `__. .. code:: ipython3 @@ -315,10 +321,10 @@ You can find more information in `Optimum Intel documentation `__ +subpackage contains definitions of models for addressing different +tasks, including: image classification, pixelwise semantic segmentation, +object detection, instance segmentation, person keypoint detection, +video classification, and optical flow. Throughout this notebook we will +show how to use one of them. The LRASPP model is based on the `Searching +for MobileNetV3 `__ paper. According +to the paper, Searching for MobileNetV3, LR-ASPP or Lite Reduced Atrous +Spatial Pyramid Pooling has a lightweight and efficient segmentation +decoder architecture. he model is pre-trained on the `MS +COCO `__ dataset. Instead of training on +all 80 classes, the segmentation model has been trained on 20 classes +from the `PASCAL VOC `__ +dataset: **background, aeroplane, bicycle, bird, boat, bottle, bus, car, +cat, chair, cow, dining table, dog, horse, motorbike, person, potted +plant, sheep, sofa, train, tv monitor** + +More information about the model is available in the `torchvision +documentation `__ + +**Table of contents:** + +- `Prerequisites <#prerequisites>`__ +- `Get a test image <#get-a-test-image>`__ +- `Download and prepare a model <#download-and-prepare-a-model>`__ +- `Define a preprocessing and prepare an input + data <#define-a-preprocessing-and-prepare-an-input-data>`__ +- `Run an inference on the PyTorch + model <#run-an-inference-on-the-pytorch-model>`__ +- `Convert the original model to OpenVINO IR + Format <#convert-the-original-model-to-openvino-ir-format>`__ +- `Run an inference on the OpenVINO + model <#run-an-inference-on-the-openvino-model>`__ +- `Show results <#show-results>`__ +- `Show results for the OpenVINO IR + model <#show-results-for-the-openvino-ir-model>`__ + +Prerequisites +------------------------------------------------------- + +.. code:: ipython3 + + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision + %pip install -q matplotlib + %pip install -q "openvino>=2023.2.0" + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + + +.. code:: ipython3 + + from pathlib import Path + + import openvino as ov + import torch + +Get a test image +---------------- + +First of all lets get a test +image from an open dataset. + +.. code:: ipython3 + + import urllib.request + + from torchvision.io import read_image + import torchvision.transforms as transforms + + + img_path = 'cats_image.jpeg' + urllib.request.urlretrieve( + url='https://huggingface.co/datasets/huggingface/cats-image/resolve/main/cats_image.jpeg', + filename=img_path + ) + image = read_image(img_path) + display(transforms.ToPILImage()(image)) + + + +.. image:: 125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.png + + +Download and prepare a model +---------------------------- + +Define width and height of the +image that will be used by the network during inference. According to +the input transforms function, the model is pre-trained on images with a +height of 480 and width of 640. + +.. code:: ipython3 + + IMAGE_WIDTH = 640 + IMAGE_HEIGHT = 480 + +Torchvision provides a mechanism of `listing and retrieving available +models `__. + +.. code:: ipython3 + + import torchvision.models as models + + # List available models + all_models = models.list_models() + # List of models by type + segmentation_models = models.list_models(module=models.segmentation) + + print(segmentation_models) + + +.. parsed-literal:: + + ['deeplabv3_mobilenet_v3_large', 'deeplabv3_resnet101', 'deeplabv3_resnet50', 'fcn_resnet101', 'fcn_resnet50', 'lraspp_mobilenet_v3_large'] + + +We will use ``lraspp_mobilenet_v3_large``. You can get a model by name +using +``models.get_model("lraspp_mobilenet_v3_large", weights='DEFAULT')`` or +call a `corresponding +function `__ +directly. We will use +``torchvision.models.segmentation.lraspp_mobilenet_v3_large``. You can +directly pass pre-trained model weights to the model initialization +function using weights enum +LRASPP_MobileNet_V3_Large_Weights.COCO_WITH_VOC_LABELS_V1. It is a +default weights. To get all available weights for the model you can call +``weights_enum = models.get_model_weights("lraspp_mobilenet_v3_large")``, +but there is only one for this model. + +.. code:: ipython3 + + weights = models.segmentation.LRASPP_MobileNet_V3_Large_Weights.COCO_WITH_VOC_LABELS_V1 + model = models.segmentation.lraspp_mobilenet_v3_large(weights=weights) + +Define a preprocessing and prepare an input data +------------------------------------------------ + +You can use +``torchvision.transforms`` to make a preprocessing or +use\ `preprocessing transforms from the model +wight `__. + +.. code:: ipython3 + + import numpy as np + + + preprocess = models.segmentation.LRASPP_MobileNet_V3_Large_Weights.COCO_WITH_VOC_LABELS_V1.transforms() + preprocess.resize_size = (IMAGE_HEIGHT, IMAGE_WIDTH) # change to an image size + + input_data = preprocess(image) + input_data = np.expand_dims(input_data, axis=0) + +Run an inference on the PyTorch model\ +------------------------------------------------------------------------------- + +.. code:: ipython3 + + model.eval() + with torch.no_grad(): + result_torch = model(torch.as_tensor(input_data).float())['out'] + +Convert the original model to OpenVINO IR Format +------------------------------------------------ + + + +To convert the original model to OpenVINO IR with ``FP16`` precision, +use model conversion API. The models are saved inside the current +directory. For more information on how to convert models, see this +`page `__. + +.. code:: ipython3 + + ov_model_xml_path = Path('models/ov_lraspp_model.xml') + + + if not ov_model_xml_path.exists(): + ov_model_xml_path.parent.mkdir(parents=True, exist_ok=True) + dummy_input = torch.randn(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH) + ov_model = ov.convert_model(model, example_input=dummy_input) + ov.save_model(ov_model, ov_model_xml_path) + else: + print(f"IR model {ov_model_xml_path} already exists.") + +Run an inference on the OpenVINO model\ +-------------------------------------------------------------------------------- + +Select device from dropdown list for running inference using OpenVINO + +.. code:: ipython3 + + import ipywidgets as widgets + + core = ov.Core() + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +Run an inference + +.. code:: ipython3 + + compiled_model = core.compile_model(ov_model_xml_path, device_name=device.value) + +.. code:: ipython3 + + res_ir = compiled_model(input_data)[0] + +Show results +------------ + +Confirm that the segmentation +results look as expected by comparing model predictions on the OpenVINO +IR and PyTorch models. + +You can use `pytorch +tutorial `__ +to visualize segmentation masks. Below is a simple example how to +visualize the image with a ``cat`` mask for the PyTorch model. + +.. code:: ipython3 + + import torch + import matplotlib.pyplot as plt + + import torchvision.transforms.functional as F + + + plt.rcParams["savefig.bbox"] = 'tight' + + + def show(imgs): + if not isinstance(imgs, list): + imgs = [imgs] + fix, axs = plt.subplots(ncols=len(imgs), squeeze=False) + for i, img in enumerate(imgs): + img = img.detach() + img = F.to_pil_image(img) + axs[0, i].imshow(np.asarray(img)) + axs[0, i].set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]) + +Prepare and display a cat mask. + +.. code:: ipython3 + + sem_classes = [ + '__background__', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', + 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', + 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor' + ] + sem_class_to_idx = {cls: idx for (idx, cls) in enumerate(sem_classes)} + + normalized_mask = torch.nn.functional.softmax(result_torch, dim=1) + + cat_mask = normalized_mask[0, sem_class_to_idx['cat']] + + show(cat_mask) + + + +.. image:: 125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_28_0.png + + +The +`draw_segmentation_masks() `__\ function +can be used to plots those masks on top of the original image. This +function expects the masks to be boolean masks, but our masks above +contain probabilities in [0, 1]. To get boolean masks, we can do the +following: + +.. code:: ipython3 + + class_dim = 1 + boolean_cat_mask = (normalized_mask.argmax(class_dim) == sem_class_to_idx['cat']) + +And now we can plot a boolean mask on top of the original image. + +.. code:: ipython3 + + from torchvision.utils import draw_segmentation_masks + + show(draw_segmentation_masks(image, masks=boolean_cat_mask, alpha=0.7, colors='yellow')) + + + +.. image:: 125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_32_0.png + + +Show results for the OpenVINO IR model\ +-------------------------------------------------------------------------------- + +.. code:: ipython3 + + normalized_mask = torch.nn.functional.softmax(torch.from_numpy(res_ir), dim=1) + boolean_cat_mask = (normalized_mask.argmax(class_dim) == sem_class_to_idx['cat']) + show(draw_segmentation_masks(image, masks=boolean_cat_mask, alpha=0.7, colors='yellow')) + + + +.. image:: 125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_34_0.png + diff --git a/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_28_0.png b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_28_0.png new file mode 100644 index 00000000000..ad37eba7fa0 --- /dev/null +++ b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_28_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b5a1c7f29fc97014c6bdd4eb2604ace96556a4864b7b2175b960a5ed1c62896 +size 87576 diff --git a/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_32_0.png b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_32_0.png new file mode 100644 index 00000000000..c48d430667b --- /dev/null +++ b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_32_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ae70e272473dddb3c96182bbe79ca0fe1a0d9867ea6b54e91d3322273753233 +size 385522 diff --git a/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_34_0.png b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_34_0.png new file mode 100644 index 00000000000..25e9953ece7 --- /dev/null +++ b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_34_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f1d79111e4b949ceb3bd19a51c7453f1bdae0ce5b8b21b748717a93e89891e +size 385520 diff --git a/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.jpg b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.jpg new file mode 100644 index 00000000000..161db4d6d6b --- /dev/null +++ b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4aeb8d18432656f38f690facffbfa90ad3966dc2113a675025d1231a7c5747b +size 63187 diff --git a/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.png b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.png new file mode 100644 index 00000000000..d86a9f8cfdf --- /dev/null +++ b/docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_5_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62c6c291107839fe807c457757e888c4804eb5ae7dc9459c590ccde8eb216fc0 +size 723717 diff --git a/docs/notebooks/125-lraspp-segmentation-with-output_files/index.html b/docs/notebooks/125-lraspp-segmentation-with-output_files/index.html new file mode 100644 index 00000000000..e9a0c4468b5 --- /dev/null +++ b/docs/notebooks/125-lraspp-segmentation-with-output_files/index.html @@ -0,0 +1,11 @@ + +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/125-lraspp-segmentation-with-output_files/ + +

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/125-lraspp-segmentation-with-output_files/


../
+125-lraspp-segmentation-with-output_28_0.png       07-Dec-2023 00:49               87576
+125-lraspp-segmentation-with-output_32_0.png       07-Dec-2023 00:49              385522
+125-lraspp-segmentation-with-output_34_0.png       07-Dec-2023 00:49              385520
+125-lraspp-segmentation-with-output_5_0.jpg        07-Dec-2023 00:49               63187
+125-lraspp-segmentation-with-output_5_0.png        07-Dec-2023 00:49              723717
+

+ diff --git a/docs/notebooks/126-tensorflow-hub-with-output.rst b/docs/notebooks/126-tensorflow-hub-with-output.rst index 2a66974a13c..9949a5d4a6d 100644 --- a/docs/notebooks/126-tensorflow-hub-with-output.rst +++ b/docs/notebooks/126-tensorflow-hub-with-output.rst @@ -21,7 +21,7 @@ You have the flexibility to run this tutorial notebook in its entirety or selectively execute specific sections, as each section operates independently. -**Table of contents:** +**Table of contents:** - `Image classification <#image-classification>`__ - `Install required packages <#install-required-packages>`__ @@ -38,7 +38,6 @@ independently. - `Select inference device <#select-inference-device>`__ - `Inference <#inference>`__ - Image classification -------------------- @@ -70,17 +69,15 @@ Install required packages .. code:: ipython3 %pip install -q tensorflow_hub tensorflow pillow numpy matplotlib - %pip install -q "openvino==2023.2.0.dev20230922" + %pip install -q "openvino>=2023.2.0" .. parsed-literal:: ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - onnxconverter-common 1.14.0 requires protobuf==3.20.2, but you have protobuf 4.25.0 which is incompatible. - tf2onnx 1.15.1 requires protobuf~=3.20.2, but you have protobuf 4.25.0 which is incompatible. + onnxconverter-common 1.14.0 requires protobuf==3.20.2, but you have protobuf 4.25.1 which is incompatible. + tf2onnx 1.15.1 requires protobuf~=3.20.2, but you have protobuf 4.25.1 which is incompatible. Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - openvino-dev 2023.1.0 requires openvino==2023.1.0, but you have openvino 2023.2.0.dev20230922 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -110,12 +107,12 @@ Import libraries IMAGE_SHAPE = (224, 224) IMAGE_URL, IMAGE_PATH = "https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg", "data/grace_hopper.jpg" - MODEL_URL, MODEL_PATH = "https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/classification/5", "models/mobilenet_v2_100_224.xml" + MODEL_URL, MODEL_PATH = "https://www.kaggle.com/models/google/mobilenet-v1/frameworks/tensorFlow2/variations/100-224-classification/versions/2", "models/mobilenet_v2_100_224.xml" Download the classifier ~~~~~~~~~~~~~~~~~~~~~~~ - Select a MobileNetV2 +Select a MobileNetV2 pre-trained model `from TensorFlow Hub `__ and wrap it as a Keras layer with ``hub.KerasLayer``. @@ -127,14 +124,14 @@ and wrap it as a Keras layer with ``hub.KerasLayer``. .. parsed-literal:: - 2023-11-14 23:08:14.660883: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW - 2023-11-14 23:08:14.661058: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration + 2023-12-06 23:12:55.307954: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW + 2023-12-06 23:12:55.308138: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration Download a single image to try the model on ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The input ``images`` are +The input ``images`` are expected to have color values in the range [0,1], following the `common image input conventions `__. @@ -319,7 +316,7 @@ Install required packages .. code:: ipython3 %pip install -q tensorflow tensorflow_hub "opencv-python" numpy matplotlib - %pip install -q "openvino==2023.2.0.dev20230922" + %pip install -q "openvino>=2023.2.0" .. parsed-literal:: @@ -351,7 +348,7 @@ Install required packages STYLE_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/b/b4/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg" STYLE_IMAGE_PATH = "./data/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg" - MODEL_URL = "https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2" + MODEL_URL = "https://www.kaggle.com/models/google/arbitrary-image-stylization-v1/frameworks/tensorFlow1/variations/256/versions/2" MODEL_PATH = "./models/arbitrary-image-stylization-v1-256.xml" Load the model diff --git a/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_26_0.png b/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_26_0.png index 7ed328b470c..4f1da92e528 100644 --- a/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_26_0.png +++ b/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_26_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6c0078b72e3696c4265712121725b2db212016519c0e609c0ffafdd98e1dc970 +oid sha256:3050fd069390a5410693d7649a6d666fe41504c7b1543897a83170e6cd8a6a5f size 203738 diff --git a/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_45_0.png b/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_45_0.png index 90d22510a83..f4139ed6d29 100644 --- a/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_45_0.png +++ b/docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_45_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:da6266c3c305c2278d74aa8d9e46c57f6c7d62dfacb92d2d616fc5e00e4cce6c +oid sha256:f34ce33eec0d7a7db4163c8ef99cadbba0249073fa82e923dfc21c3c30c665d0 size 538743 diff --git a/docs/notebooks/126-tensorflow-hub-with-output_files/index.html b/docs/notebooks/126-tensorflow-hub-with-output_files/index.html index 072161cf5c7..9d19c72ee8b 100644 --- a/docs/notebooks/126-tensorflow-hub-with-output_files/index.html +++ b/docs/notebooks/126-tensorflow-hub-with-output_files/index.html @@ -1,10 +1,10 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/126-tensorflow-hub-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/126-tensorflow-hub-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/126-tensorflow-hub-with-output_files/


../
-126-tensorflow-hub-with-output_11_0.jpg            15-Nov-2023 00:43               10479
-126-tensorflow-hub-with-output_11_0.png            15-Nov-2023 00:43               92843
-126-tensorflow-hub-with-output_26_0.png            15-Nov-2023 00:43              203738
-126-tensorflow-hub-with-output_45_0.png            15-Nov-2023 00:43              538743
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/126-tensorflow-hub-with-output_files/


../
+126-tensorflow-hub-with-output_11_0.jpg            07-Dec-2023 00:49               10479
+126-tensorflow-hub-with-output_11_0.png            07-Dec-2023 00:49               92843
+126-tensorflow-hub-with-output_26_0.png            07-Dec-2023 00:49              203738
+126-tensorflow-hub-with-output_45_0.png            07-Dec-2023 00:49              538743
 

diff --git a/docs/notebooks/201-vision-monodepth-with-output.rst b/docs/notebooks/201-vision-monodepth-with-output.rst index bd1172c37c0..ca1b1630ed5 100644 --- a/docs/notebooks/201-vision-monodepth-with-output.rst +++ b/docs/notebooks/201-vision-monodepth-with-output.rst @@ -55,11 +55,15 @@ Transactions on Pattern Analysis and Machine Intelligence, doi: Video <#do-inference-on-a-video-and-create-monodepth-video>`__ - `Display Monodepth Video <#display-monodepth-video>`__ -Preparation ------------------------------------------------------ +Preparation +----------- + + + +Install requirements +~~~~~~~~~~~~~~~~~~~~ + -Install requirements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 @@ -84,12 +88,14 @@ Install requirements .. parsed-literal:: - ('notebook_utils.py', ) + ('notebook_utils.py', ) -Imports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Imports +~~~~~~~ + + .. code:: ipython3 @@ -113,8 +119,13 @@ Imports from notebook_utils import download_file, load_image -Download the model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the model +~~~~~~~~~~~~~~~~~~ + + + +The model is in the `OpenVINO Intermediate Representation +(IR) `__ format. .. code:: ipython3 @@ -142,8 +153,10 @@ Download the model model/MiDaS_small.bin: 0%| | 0.00/31.6M [00:00 -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/201-vision-monodepth-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/201-vision-monodepth-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/201-vision-monodepth-with-output_files/


../
-201-vision-monodepth-with-output_18_0.png          31-Oct-2023 00:35              959858
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/201-vision-monodepth-with-output_files/


../
+201-vision-monodepth-with-output_18_0.png          07-Dec-2023 00:49              959858
 

diff --git a/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst b/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst index 3a53e7b87d6..02d4ce98e33 100644 --- a/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst +++ b/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst @@ -24,23 +24,21 @@ Segmenter `__. More about the model and its details can be found in the following paper: `Segmenter: Transformer for Semantic Segmentation `__ or in the -`repository `__. #### Table of -content: - `Get and prepare PyTorch -model <#get-and-prepare-pytorch-model>`__ - -`Prerequisites <#prerequisites>`__ - `Loading PyTorch -model <#loading-pytorch-model>`__ - `Preparing preprocessing and -visualization -functions <#preparing-preprocessing-and-visualization-functions>`__ -- `Preprocessing <#preprocessing>`__ - -`Visualization <#visualization>`__ - `Validation of inference of -original model <#validation-of-inference-of-original-model>`__ - -`Convert PyTorch model to OpenVINO Intermediate Representation -(IR) <#convert-pytorch-model-to-openvino-intermediate-representation-ir>`__ -- `Verify converted model -inference <#verify-converted-model-inference>`__ - `Select -inference device <#select-inference-device>`__ - `Benchmarking -performance of converted -model <#benchmarking-performance-of-converted-model>`__ +`repository `__. + +**Table of contents:** + +- `Get and prepare PyTorch model <#get-and-prepare-pytorch-model>`__ +- `Prerequisites <#prerequisites>`__ +- `Loading PyTorch model <#loading-pytorch-model>`__ +- `Preparing preprocessing and visualization functions <#preparing-preprocessing-and-visualization-functions>`__ +- `Preprocessing <#preprocessing>`__ +- `visualization <#visualization>`__ +- `Validation of inference of original model <#validation-of-inference-of-original-model>`__ +- `Convert PyTorch model to OpenVINO Intermediate Representation (IR) <#convert-pytorch-model-to-openvino-intermediate-representation-ir>`__ +- `Verify converted model inference <#verify-converted-model-inference>`__ +- `Select inference device <#select-inference-device>`__ +- `Benchmarking performance of converted model <#benchmarking-performance-of-converted-model>`__ .. |Segmenteer diagram| image:: https://user-images.githubusercontent.com/24582831/148507554-87eb80bd-02c7-4c31-b102-c6141e231ec8.png @@ -54,8 +52,10 @@ notebook consists of the following steps: - Validating inference of the converted model - Benchmark performance of the converted model -Get and prepare PyTorch model ------------------------------------------------------------------------ +Get and prepare PyTorch model +----------------------------- + + The first thing we’ll need to do is clone `repository `__ containing model @@ -69,14 +69,16 @@ The code from the repository already contains functions that create model and load weights, but we will need to download config and trained weights (checkpoint) file and add some additional helper functions. -Prerequisites -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prerequisites +~~~~~~~~~~~~~ + + .. code:: ipython3 # Installing requirements %pip install -q "openvino>=2023.1.0" - %pip install -q timm "mmsegmentation==0.30.0" einops "mmcv==1.7.1" "timm == 0.4.12" + %pip install -q timm "mmsegmentation==0.30.0" einops "mmcv==1.7.1" "timm == 0.4.12" --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -126,7 +128,7 @@ config for our model. Cloning into 'segmenter'... remote: Enumerating objects: 268, done. remote: Total 268 (delta 0), reused 0 (delta 0), pack-reused 268 - Receiving objects: 100% (268/268), 15.34 MiB | 3.50 MiB/s, done. + Receiving objects: 100% (268/268), 15.34 MiB | 3.51 MiB/s, done. Resolving deltas: 100% (117/117), done. @@ -159,8 +161,10 @@ config for our model. model/variant.yml: 0%| | 0.00/940 [00:00`__ @@ -204,18 +208,22 @@ Load normalization settings from config file. .. parsed-literal:: No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mmcv/__init__.py:20: UserWarning: On January 1, 2023, MMCV will release v2.0.0, in which it will remove components related to the training process and add a data transformation module. In addition, it will rename the package names mmcv to mmcv-lite and mmcv-full to mmcv. See https://github.com/open-mmlab/mmcv/blob/master/docs/en/compatibility.md for more details. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mmcv/__init__.py:20: UserWarning: On January 1, 2023, MMCV will release v2.0.0, in which it will remove components related to the training process and add a data transformation module. In addition, it will rename the package names mmcv to mmcv-lite and mmcv-full to mmcv. See https://github.com/open-mmlab/mmcv/blob/master/docs/en/compatibility.md for more details. warnings.warn( -Preparing preprocessing and visualization functions ---------------------------------------------------------------------------------------------- +Preparing preprocessing and visualization functions +--------------------------------------------------- + + Now we will define utility functions for preprocessing and visualizing the results. -Preprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing +~~~~~~~~~~~~~ + + Inference input is tensor with shape ``[1, 3, H, W]`` in ``B, C, H, W`` format, where: @@ -259,8 +267,10 @@ normalized with given mean and standard deviation provided in return im -Visualization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Visualization +~~~~~~~~~~~~~ + + Inference output contains labels assigned to each pixel, so the output in our case is ``[150, H, W]`` in ``CL, H, W`` format where: @@ -303,8 +313,10 @@ corresponding to the inferred labels. return pil_blend -Validation of inference of original model ------------------------------------------------------------------------------------ +Validation of inference of original model +----------------------------------------- + + Now that we have everything ready, we can perform segmentation on example image ``coco_hollywood.jpg``. @@ -355,8 +367,10 @@ We can see that model segments the image into meaningful parts. Since we are using tiny variant of model, the result is not as good as it is with larger models, but it already shows nice segmentation performance. -Convert PyTorch model to OpenVINO Intermediate Representation (IR) ------------------------------------------------------------------------------------------------------------- +Convert PyTorch model to OpenVINO Intermediate Representation (IR) +------------------------------------------------------------------ + + Now that we’ve verified that the inference of PyTorch model works, we will convert it to OpenVINO IR format. @@ -401,22 +415,24 @@ they are not a problem. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:69: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:69: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if H % patch_size > 0: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:71: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:71: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if W % patch_size > 0: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/vit.py:122: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/vit.py:122: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x.shape[1] != pos_embed.shape[1]: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/decoder.py:100: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/decoder.py:100: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! masks = rearrange(masks, "b (h w) n -> b n h w", h=int(GS)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if extra_h > 0: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if extra_w > 0: -Verify converted model inference --------------------------------------------------------------------------- +Verify converted model inference +-------------------------------- + + To test that model was successfully converted, we can use same inference function from original repository, but we need to make custom class. @@ -483,8 +499,10 @@ any additional custom code required to process input. Now that we have created ``SegmenterOV`` helper class, we can use it in inference function. -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -544,8 +562,10 @@ select device from dropdown list for running inference using OpenVINO As we can see, we get the same results as with original model. -Benchmarking performance of converted model -------------------------------------------------------------------------------------- +Benchmarking performance of converted model +------------------------------------------- + + Finally, use the OpenVINO `Benchmark Tool `__ @@ -590,18 +610,18 @@ to measure the inference performance of the model. [Step 2/11] Loading OpenVINO Runtime [ WARNING ] Default duration 120 seconds is used for unknown device AUTO [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.2.0-12538-e7c1344d3c3 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2023.2.0-12538-e7c1344d3c3 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 24.01 ms + [ INFO ] Read model took 22.59 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] im (node: im) : f32 / [...] / [2,3,512,512] @@ -615,7 +635,7 @@ to measure the inference performance of the model. [ INFO ] Model outputs: [ INFO ] y (node: aten::upsample_bilinear2d/Interpolate) : f32 / [...] / [2,150,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 387.83 ms + [ INFO ] Compile model took 375.79 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -647,15 +667,15 @@ to measure the inference performance of the model. [ INFO ] Fill input 'im' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 208.27 ms + [ INFO ] First inference took 209.86 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 1392 iterations - [ INFO ] Duration: 120914.63 ms + [ INFO ] Count: 1692 iterations + [ INFO ] Duration: 120618.19 ms [ INFO ] Latency: - [ INFO ] Median: 520.24 ms - [ INFO ] Average: 520.33 ms - [ INFO ] Min: 364.28 ms - [ INFO ] Max: 586.27 ms - [ INFO ] Throughput: 23.02 FPS + [ INFO ] Median: 427.65 ms + [ INFO ] Average: 426.90 ms + [ INFO ] Min: 189.03 ms + [ INFO ] Max: 509.01 ms + [ INFO ] Throughput: 28.06 FPS diff --git a/docs/notebooks/204-segmenter-semantic-segmentation-with-output_files/index.html b/docs/notebooks/204-segmenter-semantic-segmentation-with-output_files/index.html index 63465799871..7706eac7f4d 100644 --- a/docs/notebooks/204-segmenter-semantic-segmentation-with-output_files/index.html +++ b/docs/notebooks/204-segmenter-semantic-segmentation-with-output_files/index.html @@ -1,10 +1,10 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/


../
-204-segmenter-semantic-segmentation-with-output..> 31-Oct-2023 00:35               72352
-204-segmenter-semantic-segmentation-with-output..> 31-Oct-2023 00:35              909669
-204-segmenter-semantic-segmentation-with-output..> 31-Oct-2023 00:35               72372
-204-segmenter-semantic-segmentation-with-output..> 31-Oct-2023 00:35              909654
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/


../
+204-segmenter-semantic-segmentation-with-output..> 07-Dec-2023 00:49               72352
+204-segmenter-semantic-segmentation-with-output..> 07-Dec-2023 00:49              909669
+204-segmenter-semantic-segmentation-with-output..> 07-Dec-2023 00:49               72372
+204-segmenter-semantic-segmentation-with-output..> 07-Dec-2023 00:49              909654
 

diff --git a/docs/notebooks/210-slowfast-video-recognition-with-output.rst b/docs/notebooks/210-slowfast-video-recognition-with-output.rst index db2faf79514..0f2f5e02293 100644 --- a/docs/notebooks/210-slowfast-video-recognition-with-output.rst +++ b/docs/notebooks/210-slowfast-video-recognition-with-output.rst @@ -43,29 +43,34 @@ This tutorial consists of the following steps - `Prepare PyTorch Model <#prepare-pytorch-model>`__ - - `Install necessary - packages <#install-necessary-packages>`__ + - `Install necessary packages <#install-necessary-packages>`__ - `Imports and Settings <#imports-and-settings>`__ - `Export to ONNX <#export-to-onnx>`__ -- `Convert ONNX to OpenVINO™ Intermediate +- `Convert ONNX to OpenVINO Intermediate Representation <#convert-onnx-to-openvino-intermediate-representation>`__ - `Select inference device <#select-inference-device>`__ - `Verify Model Inference <#verify-model-inference>`__ -Prepare PyTorch Model ---------------------------------------------------------------- +Prepare PyTorch Model +--------------------- + + + +Install necessary packages +~~~~~~~~~~~~~~~~~~~~~~~~~~ + -Install necessary packages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 !pip install -q "openvino>=2023.1.0" - !pip install -q fvcore + !pip install -q fvcore --extra-index-url https://download.pytorch.org/whl/cpu + +Imports and Settings +~~~~~~~~~~~~~~~~~~~~ + -Imports and Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 @@ -709,8 +714,10 @@ inference using the same. The top 5 predictions can be seen below. Predicted labels: archery, throwing axe, playing paintball, golf driving, riding or walking with horse -Export to ONNX --------------------------------------------------------- +Export to ONNX +-------------- + + Now that we have obtained our trained model and checked inference with it, we export the PyTorch model to Open Neural Network Exchange(ONNX) @@ -733,8 +740,10 @@ quantization. export_params=True, ) -Convert ONNX to OpenVINO Intermediate Representation ----------------------------------------------------------------------------------------------- +Convert ONNX to OpenVINO Intermediate Representation +---------------------------------------------------- + + Now that our ONNX model is ready, we can convert it to IR format. In this format, the network is represented using two files: an ``xml`` file @@ -765,8 +774,10 @@ using the ``weights`` parameter. # read converted model conv_model = core.read_model(str(IR_PATH)) -Select inference device ------------------------------------------------------------------ +Select inference device +----------------------- + + select device from dropdown list for running inference using OpenVINO @@ -797,8 +808,10 @@ select device from dropdown list for running inference using OpenVINO # load model on device compiled_model = core.compile_model(model=conv_model, device_name=device.value) -Verify Model Inference ----------------------------------------------------------------- +Verify Model Inference +---------------------- + + Using the compiled model, we run inference on the same sample video and print the top 5 predictions again. diff --git a/docs/notebooks/216-attention-center-with-output.rst b/docs/notebooks/216-attention-center-with-output.rst new file mode 100644 index 00000000000..39b3c2bb2e0 --- /dev/null +++ b/docs/notebooks/216-attention-center-with-output.rst @@ -0,0 +1,347 @@ +The attention center model with OpenVINO™ +========================================= + +This notebook demonstrates how to use the `attention center +model `__ with +OpenVINO. This model is in the `TensorFlow Lite +format `__, which is supported in +OpenVINO now by TFLite frontend. + +Eye tracking is commonly used in visual neuroscience and cognitive +science to answer related questions such as visual attention and +decision making. Computational models that predict where to look have +direct applications to a variety of computer vision tasks. The attention +center model takes an RGB image as input and return a 2D point as +output. This 2D point is the predicted center of human attention on the +image i.e. the most salient part of images, on which people pay +attention fist to. This allows find the most visually salient regions +and handle it as early as possible. For example, it could be used for +the latest generation image format (such as `JPEG +XL `__), which supports encoding the +parts that you pay attention to fist. It can help to improve user +experience, image will appear to load faster. + +Attention center model architecture is: > The attention center model is +a deep neural net, which takes an image as input, and uses a pre-trained +classification network, e.g, ResNet, MobileNet, etc., as the backbone. +Several intermediate layers that output from the backbone network are +used as input for the attention center prediction module. These +different intermediate layers contain different information e.g., +shallow layers often contain low level information like +intensity/color/texture, while deeper layers usually contain higher and +more semantic information like shape/object. All are useful for the +attention prediction. The attention center prediction applies +convolution, deconvolution and/or resizing operator together with +aggregation and sigmoid function to generate a weighting map for the +attention center. And then an operator (the Einstein summation operator +in our case) can be applied to compute the (gravity) center from the +weighting map. An L2 norm between the predicted attention center and the +ground-truth attention center can be computed as the training loss. +Source: `Google AI blog +post `__. + +.. figure:: https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjxLCDJHzJNjB_von-vFlq8TJJFA41aB85T-QE3ZNxW8kshAf3HOEyIEJ4uggXjbJmZhsdj7j6i6mvvmXtyaxXJPm3JHuKILNRTPfX9KvICbFBRD8KNuDVmLABzYuhQci3BT2BqV-wM54IxaoAV1YDBbnpJC92UZfEBGvakLusiqND2AaPpWPr2gJV1/s1600/image4.png + :alt: drawing + + drawing + +The attention center model has been trained with images from the `COCO +dataset `__ annotated with saliency from +the `SALICON dataset `__. + +**Table of contents:** + + +- `Imports <#imports>`__ +- `Download the attention-center + model <#download-the-attention-center-model>`__ + + - `Convert Tensorflow Lite model to OpenVINO IR + format <#convert-tensorflow-lite-model-to-openvino-ir-format>`__ + +- `Select inference device <#select-inference-device>`__ +- `Prepare image to use with attention-center + model <#prepare-image-to-use-with-attention-center-model>`__ +- `Load input image <#load-input-image>`__ +- `Get result with OpenVINO IR + model <#get-result-with-openvino-ir-model>`__ + +.. code:: ipython3 + + %pip install "openvino>=2023.2.0" + + +.. parsed-literal:: + + Requirement already satisfied: openvino>=2023.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2023.2.0) + Requirement already satisfied: numpy>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.2.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.2.0) (2023.2.1) + DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + Note: you may need to restart the kernel to use updated packages. + + +Imports +------- + + + +.. code:: ipython3 + + import cv2 + + import numpy as np + import tensorflow as tf + from pathlib import Path + import matplotlib.pyplot as plt + + import openvino as ov + + +.. parsed-literal:: + + 2023-12-06 23:32:10.485958: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 23:32:10.520826: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2023-12-06 23:32:11.062803: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + +Download the attention-center model +----------------------------------- + + + +Download the model as part of `attention-center +repo `__. The repo +include model in folder ``./model``. + +.. code:: ipython3 + + if not Path('./attention-center').exists(): + ! git clone https://github.com/google/attention-center + + +.. parsed-literal:: + + Cloning into 'attention-center'... + remote: Enumerating objects: 168, done. + remote: Counting objects: 100% (168/168), done. + remote: Compressing objects: 100% (132/132), done. + remote: Total 168 (delta 73), reused 114 (delta 28), pack-reused 0 + Receiving objects: 100% (168/168), 26.22 MiB | 3.34 MiB/s, done. + Resolving deltas: 100% (73/73), done. + + +Convert Tensorflow Lite model to OpenVINO IR format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +The attention-center model is pre-trained model in TensorFlow Lite +format. In this Notebook the model will be converted to OpenVINO IR +format with model conversion API. For more information about model +conversion, see this +`page `__. +This step is also skipped if the model is already converted. + +Also TFLite models format is supported in OpenVINO by TFLite frontend, +so the model can be passed directly to ``core.read_model()``. You can +find example in +`002-openvino-api `__. + +.. code:: ipython3 + + tflite_model_path = Path("./attention-center/model/center.tflite") + + ir_model_path = Path("./model/ir_center_model.xml") + + core = ov.Core() + + if not ir_model_path.exists(): + model = ov.convert_model(tflite_model_path, input=[('image:0', [1,480,640,3], ov.Type.f32)]) + ov.save_model(model, ir_model_path) + print("IR model saved to {}".format(ir_model_path)) + else: + print("Read IR model from {}".format(ir_model_path)) + model = core.read_model(ir_model_path) + + +.. parsed-literal:: + + IR model saved to model/ir_center_model.xml + + +Select inference device +----------------------- + + + +select device from dropdown list for running inference using OpenVINO + +.. code:: ipython3 + + import ipywidgets as widgets + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +.. code:: ipython3 + + if "GPU" in device.value: + core.set_property(device_name=device.value, properties={'INFERENCE_PRECISION_HINT': ov.Type.f32}) + compiled_model = core.compile_model(model=model, device_name=device.value) + +Prepare image to use with attention-center model +------------------------------------------------ + + + +The attention-center model takes an RGB image with shape (480, 640) as +input. + +.. code:: ipython3 + + class Image(): + def __init__(self, model_input_image_shape, image_path=None, image=None): + self.model_input_image_shape = model_input_image_shape + self.image = None + self.real_input_image_shape = None + + if image_path is not None: + self.image = cv2.imread(str(image_path)) + self.real_input_image_shape = self.image.shape + elif image is not None: + self.image = image + self.real_input_image_shape = self.image.shape + else: + raise Exception("Sorry, image can't be found, please, specify image_path or image") + + def prepare_image_tensor(self): + rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB) + resized_image = cv2.resize(rgb_image, (self.model_input_image_shape[1], self.model_input_image_shape[0])) + + image_tensor = tf.constant(np.expand_dims(resized_image, axis=0), + dtype=tf.float32) + return image_tensor + + def scalt_center_to_real_image_shape(self, predicted_center): + new_center_y = round(predicted_center[0] * self.real_input_image_shape[1] / self.model_input_image_shape[1]) + new_center_x = round(predicted_center[1] * self.real_input_image_shape[0] / self.model_input_image_shape[0]) + return (int(new_center_y), int(new_center_x)) + + def draw_attention_center_point(self, predicted_center): + image_with_circle = cv2.circle(self.image, + predicted_center, + radius=10, + color=(3, 3, 255), + thickness=-1) + return image_with_circle + + def print_image(self, predicted_center=None): + image_to_print = self.image + if predicted_center is not None: + image_to_print = self.draw_attention_center_point(predicted_center) + + plt.imshow(cv2.cvtColor(image_to_print, cv2.COLOR_BGR2RGB)) + +Load input image +---------------- + + + +Upload input image using file loading button + +.. code:: ipython3 + + import ipywidgets as widgets + + load_file_widget = widgets.FileUpload( + accept="image/*", multiple=False, description="Image file", + ) + + load_file_widget + + + + +.. parsed-literal:: + + FileUpload(value=(), accept='image/*', description='Image file') + + + +.. code:: ipython3 + + import io + import PIL + from urllib.request import urlretrieve + + img_path = Path("data/coco.jpg") + img_path.parent.mkdir(parents=True, exist_ok=True) + urlretrieve( + "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", + img_path, + ) + + # read uploaded image + image = PIL.Image.open(io.BytesIO(list(load_file_widget.value.values())[-1]['content'])) if load_file_widget.value else PIL.Image.open(img_path) + image.convert("RGB") + + input_image = Image((480, 640), image=(np.ascontiguousarray(image)[:, :, ::-1]).astype(np.uint8)) + image_tensor = input_image.prepare_image_tensor() + input_image.print_image() + + +.. parsed-literal:: + + 2023-12-06 23:32:25.308665: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW + 2023-12-06 23:32:25.308704: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07 + 2023-12-06 23:32:25.308709: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07 + 2023-12-06 23:32:25.308855: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2 + 2023-12-06 23:32:25.308869: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3 + 2023-12-06 23:32:25.308873: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration + + + +.. image:: 216-attention-center-with-output_files/216-attention-center-with-output_15_1.png + + +Get result with OpenVINO IR model +--------------------------------- + + + +.. code:: ipython3 + + output_layer = compiled_model.output(0) + + # make inference, get result in input image resolution + res = compiled_model([image_tensor])[output_layer] + # scale point to original image resulution + predicted_center = input_image.scalt_center_to_real_image_shape(res[0]) + print(f'Prediction attention center point {predicted_center}') + input_image.print_image(predicted_center) + + +.. parsed-literal:: + + Prediction attention center point (292, 277) + + + +.. image:: 216-attention-center-with-output_files/216-attention-center-with-output_17_1.png + diff --git a/docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_15_1.png b/docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_15_1.png new file mode 100644 index 00000000000..3f302494abc --- /dev/null +++ b/docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_15_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd4338c6c163e7693885ce544e8c9cd2aecedf3b136fa295e22877f37b5634c +size 387941 diff --git a/docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_17_1.png b/docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_17_1.png new file mode 100644 index 00000000000..649b33ce6c0 --- /dev/null +++ b/docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_17_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03e8883f23b57427be0e00338ba25397c859132d9c0891ff537cc9b355a3ccca +size 387905 diff --git a/docs/notebooks/216-attention-center-with-output_files/index.html b/docs/notebooks/216-attention-center-with-output_files/index.html new file mode 100644 index 00000000000..f332a8b4e96 --- /dev/null +++ b/docs/notebooks/216-attention-center-with-output_files/index.html @@ -0,0 +1,8 @@ + +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/216-attention-center-with-output_files/ + +

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/216-attention-center-with-output_files/


../
+216-attention-center-with-output_15_1.png          07-Dec-2023 00:49              387941
+216-attention-center-with-output_17_1.png          07-Dec-2023 00:49              387905
+

+ diff --git a/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst b/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst index dbf2a1ba91d..4b7c9d1ac08 100644 --- a/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst +++ b/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst @@ -22,7 +22,7 @@ The notebook guides you through the entire process of creating a parallel book: from obtaining raw texts to building a visualization of aligned sentences. Here is the pipeline diagram: -|image1| +|image0| Visualizing the result allows you to identify areas for improvement in the pipeline steps, as indicated in the diagram. @@ -37,10 +37,9 @@ Prerequisites - ``seaborn`` - for alignment matrix visualization - ``ipywidgets`` - for displaying HTML and JS output in the notebook +**Table of contents:** -**Table of contents**: - -- `Get Books <#get-books>`__ +- `Get Books <#get-books>`__ - `Clean Text <#clean-text>`__ - `Split Text <#split-text>`__ - `Get Sentence Embeddings <#get-sentence-embeddings>`__ @@ -48,23 +47,22 @@ Prerequisites - `Optimize the Model with OpenVINO <#optimize-the-model-with-openvino>`__ -- `Calculate Sentence - Alignment <#calculate-sentence-alignment>`__ -- `Postprocess Sentence - Alignment <#postprocess-sentence-alignment>`__ -- `Visualize Sentence - Alignment <#visualize-sentence-alignment>`__ +- `Calculate Sentence Alignment <#calculate-sentence-alignment>`__ +- `Postprocess Sentence Alignment <#postprocess-sentence-alignment>`__ +- `Visualize Sentence Alignment <#visualize-sentence-alignment>`__ - `Speed up Embeddings Computation <#speed-up-embeddings-computation>`__ -.. |image1| image:: https://user-images.githubusercontent.com/51917466/254582697-18f3ab38-e264-4b2c-a088-8e54b855c1b2.png +.. |image0| image:: https://user-images.githubusercontent.com/51917466/254582697-18f3ab38-e264-4b2c-a088-8e54b855c1b2.png .. code:: ipython3 - !pip install -q --extra-index-url https://download.pytorch.org/whl/cpu requests pysbd transformers[torch] "openvino>=2023.1.0" seaborn ipywidgets + !pip install -q --extra-index-url https://download.pytorch.org/whl/cpu requests pysbd transformers[torch] "openvino>=2023.1.0" matplotlib seaborn ipywidgets Get Books ---------------------------------------------------- +--------- + + The first step is to get the books that we will be working with. For this notebook, we will use English and German versions of Anna Karenina @@ -92,7 +90,9 @@ To get the texts, we will pass the IDs to the request.raise_for_status() book_metadata = request.json() - book_url = book_metadata["formats"]["text/plain"] + text_format_key = "text/plain" + text_plain = [k for k in book_metadata["formats"] if k.startswith(text_format_key)] + book_url = book_metadata["formats"][text_plain[0]] return requests.get(book_url).text @@ -135,7 +135,7 @@ Let’s check that we got the right books by showing a part of the texts: - \*\*\* START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA \*\*\* + *** START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA *** [Illustration] @@ -189,7 +189,7 @@ which in a raw format looks like this: .. parsed-literal:: - '\ufeffThe Project Gutenberg eBook of Anna Karenina\r\n \r\nThis ebook is for the use of anyone anywhere in the United States and\r\nmost other parts of the world at no cost and with almost no restrictions\r\nwhatsoever. You may copy it, give it away or re-use it under the terms\r\nof the Project Gutenberg License included with this ebook or online\r\nat www.gutenberg.org. If you are not located in the United States,\r\nyou will have to check the laws of the country where you are located\r\nbefore using this eBook.\r\n\r\nTitle: Anna Karenina\r\n\r\n\r\nAuthor: graf Leo Tolstoy\r\n\r\nTranslator: Constance Garnett\r\n\r\nRelease date: July 1, 1998 [eBook #1399]\r\n Most recently updated: April 9, 2023\r\n\r\nLanguage: English\r\n\r\n\r\n\r\n\* START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA \*\r\n[Illustration]\r\n\r\n\r\n\r\n\r\n ANNA KARENINA \r\n\r\n by Leo Tolstoy \r\n\r\n Translated by Constance Garnett \r\n\r\nContents\r\n\r\n\r\n PART ONE\r\n PART TWO\r\n PART THREE\r\n PART FOUR\r\n PART FIVE\r\n PART SIX\r\n PART SEVEN\r\n PART EIGHT\r\n\r\n\r\n\r\n\r\nPART ONE\r\n\r\nChapter 1\r\n\r\n\r\nHappy families are all alike; every unhappy family is unhappy in its\r\nown way.\r\n\r\nEverything was in confusion in the Oblonskys’ house. The wife had\r\ndiscovered that the husband was carrying on an intrigue with a French\r\ngirl, who had been a governess in their family, and she had announced\r\nto her husband that she could not go on living in the same house with\r\nhim. This position of affairs had now lasted three days, and not only\r\nthe husband and wife themselves, but all the me' + '\ufeffThe Project Gutenberg eBook of Anna Karenina\r\n \r\nThis ebook is for the use of anyone anywhere in the United States and\r\nmost other parts of the world at no cost and with almost no restrictions\r\nwhatsoever. You may copy it, give it away or re-use it under the terms\r\nof the Project Gutenberg License included with this ebook or online\r\nat www.gutenberg.org. If you are not located in the United States,\r\nyou will have to check the laws of the country where you are located\r\nbefore using this eBook.\r\n\r\nTitle: Anna Karenina\r\n\r\n\r\nAuthor: graf Leo Tolstoy\r\n\r\nTranslator: Constance Garnett\r\n\r\nRelease date: July 1, 1998 [eBook #1399]\r\n Most recently updated: April 9, 2023\r\n\r\nLanguage: English\r\n\r\n\r\n\r\n\*\*\* START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA \*\*\*\r\n[Illustration]\r\n\r\n\r\n\r\n\r\n ANNA KARENINA \r\n\r\n by Leo Tolstoy \r\n\r\n Translated by Constance Garnett \r\n\r\nContents\r\n\r\n\r\n PART ONE\r\n PART TWO\r\n PART THREE\r\n PART FOUR\r\n PART FIVE\r\n PART SIX\r\n PART SEVEN\r\n PART EIGHT\r\n\r\n\r\n\r\n\r\nPART ONE\r\n\r\nChapter 1\r\n\r\n\r\nHappy families are all alike; every unhappy family is unhappy in its\r\nown way.\r\n\r\nEverything was in confusion in the Oblonskys’ house. The wife had\r\ndiscovered that the husband was carrying on an intrigue with a French\r\ngirl, who had been a governess in their family, and she had announced\r\nto her husband that she could not go on living in the same house with\r\nhim. This position of affairs had now lasted three days, and not only\r\nthe husband and wife themselves, but all the me' @@ -202,12 +202,14 @@ which in a raw format looks like this: .. parsed-literal:: - '\ufeffThe Project Gutenberg eBook of Anna Karenina, 1. Band\r\n \r\nThis ebook is for the use of anyone anywhere in the United States and\r\nmost other parts of the world at no cost and with almost no restrictions\r\nwhatsoever. You may copy it, give it away or re-use it under the terms\r\nof the Project Gutenberg License included with this ebook or online\r\nat www.gutenberg.org. If you are not located in the United States,\r\nyou will have to check the laws of the country where you are located\r\nbefore using this eBook.\r\n\r\nTitle: Anna Karenina, 1. Band\r\n\r\n\r\nCreator: graf Leo Tolstoy\r\n\r\nRelease date: February 18, 2014 [eBook #44956]\r\n\r\nLanguage: German\r\n\r\n\r\n\r\n\*\*\* START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA, 1. BAND \*\r\n\r\n\r\n\r\nProduced by Norbert H. Langkau, Jens Nordmann and the\r\nOnline Distributed Proofreading Team at http://www.pgdp.net\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n Anna Karenina.\r\n\r\n\r\n Roman aus dem Russischen\r\n\r\n des\r\n\r\n Grafen Leo N. Tolstoi.\r\n\r\n\r\n\r\n Nach der siebenten Auflage übersetzt\r\n\r\n von\r\n\r\n Hans Moser.\r\n\r\n\r\n Erster Band.\r\n\r\n\r\n\r\n Leipzig\r\n\r\n Druck und Verlag von Philipp Reclam jun.\r\n\r\n \* \* \* \* \*\r\n\r\n\r\n\r\n\r\n Erster Teil.\r\n\r\n »Die' + 'The Project Gutenberg EBook of Anna Karenina, 1. Band, by Leo N. Tolstoi\r\n\r\nThis eBook is for the use of anyone anywhere at no cost and with\r\nalmost no restrictions whatsoever. You may copy it, give it away or\r\nre-use it under the terms of the Project Gutenberg License included\r\nwith this eBook or online at www.gutenberg.org\r\n\r\n\r\nTitle: Anna Karenina, 1. Band\r\n\r\nAuthor: Leo N. Tolstoi\r\n\r\nRelease Date: February 18, 2014 [EBook #44956]\r\n\r\nLanguage: German\r\n\r\nCharacter set encoding: ISO-8859-1\r\n\r\n\*\*\* START OF THIS PROJECT GUTENBERG EBOOK ANNA KARENINA, 1. BAND \*\*\*\r\n\r\n\r\n\r\n\r\nProduced by Norbert H. Langkau, Jens Nordmann and the\r\nOnline Distributed Proofreading Team at http://www.pgdp.net\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n Anna Karenina.\r\n\r\n\r\n Roman aus dem Russischen\r\n\r\n des\r\n\r\n Grafen Leo N. Tolstoi.\r\n\r\n\r\n\r\n Nach der siebenten Auflage übersetzt\r\n\r\n von\r\n\r\n Hans Moser.\r\n\r\n\r\n Erster Band.\r\n\r\n\r\n\r\n Leipzig\r\n\r\n Druck und Verlag von Philipp Reclam jun.\r\n\r\n \* \* \* \* \*\r\n\r\n\r\n\r\n\r\n Erster Teil.\r\n\r\n »Die Rache ist mein, ich will vergelten.«\r\n\r\n 1.\r\n\r\n\r\nAlle glücklichen Familien sind einander ähnlich; jede unglücklich' Clean Text ----------------------------------------------------- +---------- + + The downloaded books may contain service information before and after the main text. The text might have different formatting styles and @@ -222,11 +224,11 @@ underscores for potential emphasis or italicization: The next stages of the pipeline will be difficult to complete without cleaning and normalizing the text. Since formatting may differ, manual work is required at this stage. For example, the main content in the -German version is enclosed in ``* * * * *``, so +German version is enclosed in ``* * * * *``, so it is safe to remove everything before the first occurrence and after the last occurrence of these asterisks. - Hint: There are text-cleaning libraries that clean up common + **Hint**: There are text-cleaning libraries that clean up common flaws. If the source of the text is known, you can look for a library designed for that source, for example `gutenberg_cleaner `__. @@ -243,7 +245,7 @@ the last occurrence of these asterisks. start_pattern_en = r"\nPART ONE" anna_karenina_en = re.split(start_pattern_en, anna_karenina_en)[1].strip() - end_pattern_en = "* END OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA *" + end_pattern_en = "*** END OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA ***" anna_karenina_en = anna_karenina_en.split(end_pattern_en)[0].strip() .. code:: ipython3 @@ -333,7 +335,9 @@ needed. Split Text ----------------------------------------------------- +---------- + + Dividing text into sentences is a challenging task in text processing. The problem is called `sentence boundary @@ -345,7 +349,7 @@ code `__, as the rules for splitting text into sentences may vary for different languages. - Hint: The ``book_metadata`` obtained from the Gutendex contains + **Hint**: The ``book_metadata`` obtained from the Gutendex contains the language code as well, enabling automation of this part of the pipeline. @@ -373,7 +377,9 @@ languages. Get Sentence Embeddings ------------------------------------------------------------------ +----------------------- + + The next step is to transform sentences into vector representations. Transformer encoder models, like BERT, provide high-quality embeddings @@ -393,12 +399,12 @@ languages. It has the same architecture as the BERT model but has been trained on a different task: to produce identical embeddings for translation pairs. -|image0| +|image01| This makes LaBSE a great choice for our task and it can be reused for different language pairs still producing good results. -.. |image0| image:: https://user-images.githubusercontent.com/51917466/254582913-51531880-373b-40cb-bbf6-1965859df2eb.png%22 +.. |image01| image:: https://user-images.githubusercontent.com/51917466/254582913-51531880-373b-40cb-bbf6-1965859df2eb.png .. code:: ipython3 @@ -414,15 +420,6 @@ different language pairs still producing good results. pt_model = AutoModel.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) - -.. parsed-literal:: - - 2023-09-15 18:53:46.819925: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-09-15 18:53:46.859715: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-09-15 18:53:47.576875: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - The model has two outputs: ``last_hidden_state`` and ``pooler_output``. For generating embeddings, you can use either the first vector from the ``last_hidden_state``, which corresponds to the special ``[CLS]`` token, @@ -447,7 +444,7 @@ best fit. return np.vstack(embeddings) else: embeddings = [ - embedding_model(tokenizer(sent, return_tensors="pt"))[ + embedding_model(**tokenizer(sent, return_tensors="pt"))[ "last_hidden_state" ][0][0] for sent in tqdm(sentences, disable=disable_tqdm) @@ -472,7 +469,9 @@ best fit. Optimize the Model with OpenVINO -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The LaBSE model is quite large and can be slow to infer on some hardware, so let’s optimize it with OpenVINO. `Model conversion Python @@ -500,40 +499,6 @@ The converted model must be compiled for the target device using the embeddings_de = get_embeddings(sentences_de, compiled_model) -.. parsed-literal:: - - WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - - -.. parsed-literal:: - - [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - - -.. parsed-literal:: - - INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino - WARNING:nncf:NNCF provides best results with torch==2.0.1, while current torch version is 1.13.1+cu117. If you encounter issues, consider switching to torch==2.0.1 - huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... - To disable this warning, you can either: - - Avoid using `tokenizers` before the fork if possible - - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) - huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... - To disable this warning, you can either: - - Avoid using `tokenizers` before the fork if possible - - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) - huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... - To disable this warning, you can either: - - Avoid using `tokenizers` before the fork if possible - - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) - - -.. parsed-literal:: - - /home/ea/work/ov_venv/lib/python3.8/site-packages/torch/jit/annotations.py:309: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either. - warnings.warn("TorchScript will treat type annotations of Tensor " - - .. parsed-literal:: @@ -566,7 +531,9 @@ model predictions remain within an acceptable tolerance: Calculate Sentence Alignment ----------------------------------------------------------------------- +---------------------------- + + With the embedding matrices from the previous step, we can calculate the alignment: 1. Calculate sentence similarity between each pair of @@ -691,7 +658,9 @@ will be lists of German sentence numbers. Postprocess Sentence Alignment ------------------------------------------------------------------------- +------------------------------ + + There are several gaps in the resulting alignment, such as English sentence #14 not mapping to any German sentence. Here are some possible @@ -716,7 +685,9 @@ Most likely, English sentence 14 is part of either German sentence 17 or suitable alignment. Visualize Sentence Alignment ----------------------------------------------------------------------- +---------------------------- + + To evaluate the final alignment and choose the best way to improve the results of the pipeline, we will create an interactive table with HTML @@ -875,7 +846,9 @@ To read the model from disk, use the ``read_model`` method of the ov_model = core.read_model(ov_model_path) Speed up Embeddings Computation -------------------------------------------------------------------------- +------------------------------- + + Let’s see how we can speed up the most computationally complex part of the pipeline - getting embeddings. You might wonder why, when using @@ -960,7 +933,7 @@ advance and fill it in as the inference requests are executed. Let’s compare the models and plot the results. - Note: To get a more accurate benchmark, use the `Benchmark Python + **Note**: To get a more accurate benchmark, use the `Benchmark Python Tool `__ .. code:: ipython3 @@ -1054,12 +1027,6 @@ Let’s compare the models and plot the results. ylabel="Sentences Per Second", title=f"Sentence Embeddings Benchmark\n{cpu_name}" ) perf_ratio = benchmark_dataframe.mean() / benchmark_dataframe.mean()[0] - plot.bar_label( - plot.containers[0], - labels=[f"×{ratio:.2f}" for ratio in perf_ratio], - color="white", - label_type="center", - ) plot.spines["right"].set_visible(False) plot.spines["top"].set_visible(False) plot.spines["left"].set_visible(False) diff --git a/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_31_0.png b/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_31_0.png index b2ceabfb965..d22e0ec2318 100644 --- a/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_31_0.png +++ b/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_31_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ce28e8a2906c5681f16ca683fd2ed51f24efdf0e16b3245ddafe16b9556e28f7 +oid sha256:6eb8ff85598a36eb189f49ad757496939d0c9e57096300f63764a13bbf205cbf size 24464 diff --git a/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_48_0.png b/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_48_0.png index 02bdaf9d68f..ecf3f13078b 100644 --- a/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_48_0.png +++ b/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/220-cross-lingual-books-alignment-with-output_48_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4597f3d5412fcd4194f85bb9cd30734052a8de052a11eba2b57b741f89bdbeb -size 32175 +oid sha256:86749919eec1fdc021522f51fb2e63fbdc25cde3c0e3e5a3dcaff82f4bf66628 +size 29527 diff --git a/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/index.html b/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/index.html index 687e04a344b..d9e7525e5f9 100644 --- a/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/index.html +++ b/docs/notebooks/220-cross-lingual-books-alignment-with-output_files/index.html @@ -1,8 +1,8 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/220-cross-lingual-books-alignment-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/220-cross-lingual-books-alignment-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/220-cross-lingual-books-alignment-with-output_files/


../
-220-cross-lingual-books-alignment-with-output_3..> 31-Oct-2023 00:35               24464
-220-cross-lingual-books-alignment-with-output_4..> 31-Oct-2023 00:35               32175
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/220-cross-lingual-books-alignment-with-output_files/


../
+220-cross-lingual-books-alignment-with-output_3..> 07-Dec-2023 00:49               24464
+220-cross-lingual-books-alignment-with-output_4..> 07-Dec-2023 00:49               29527
 

diff --git a/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst b/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst index 53fd6f3522b..d976db28c19 100644 --- a/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst +++ b/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst @@ -23,14 +23,12 @@ effective, showing strong performance on par or even better than state of the art. **Table of contents:** ---- -- `Imports <#imports>`__ +- `Imports <#imports>`__ - `Prepare the Model <#prepare-the-model>`__ - `Data Processing Module <#data-processing-module>`__ -- `Visualize the original 3D - data <#visualize-the-original-d-data>`__ +- `Visualize the original 3D data <#visualize-the-original-d-data>`__ - `Run inference <#run-inference>`__ - `Select inference device <#select-inference-device>`__ @@ -46,8 +44,10 @@ of the art. Note: you may need to restart the kernel to use updated packages. -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -66,8 +66,10 @@ Imports from notebook_utils import download_file -Prepare the Model ------------------------------------------------------------ +Prepare the Model +----------------- + + Download the pre-trained PointNet ONNX model. This pre-trained model is provided by `axinc-ai `__, and you can @@ -109,8 +111,10 @@ API, see this model = core.read_model(model=ir_model_xml) -Data Processing Module ----------------------------------------------------------------- +Data Processing Module +---------------------- + + .. code:: ipython3 @@ -164,8 +168,10 @@ Data Processing Module return ax -Visualize the original 3D data ------------------------------------------------------------------------- +Visualize the original 3D data +------------------------------ + + The point cloud data can be downloaded from `ShapeNet `__, @@ -197,12 +203,20 @@ chair for example. data/chair.pts: 0%| | 0.00/69.2k [00:00 -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/224-3D-segmentation-point-clouds-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/224-3D-segmentation-point-clouds-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/224-3D-segmentation-point-clouds-with-output_files/


../
-224-3D-segmentation-point-clouds-with-output_11..> 31-Oct-2023 00:35              209355
-224-3D-segmentation-point-clouds-with-output_16..> 31-Oct-2023 00:35              222552
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/224-3D-segmentation-point-clouds-with-output_files/


../
+224-3D-segmentation-point-clouds-with-output_11..> 07-Dec-2023 00:49              207243
+224-3D-segmentation-point-clouds-with-output_16..> 07-Dec-2023 00:49              220281
 

diff --git a/docs/notebooks/226-yolov7-optimization-with-output.rst b/docs/notebooks/226-yolov7-optimization-with-output.rst index 9fa5d0ad5c3..1e45938931c 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output.rst +++ b/docs/notebooks/226-yolov7-optimization-with-output.rst @@ -57,8 +57,7 @@ The tutorial consists of the following steps: - `Download dataset <#download-dataset>`__ - `Create dataloader <#create-dataloader>`__ - - `Define validation - function <#define-validation-function>`__ + - `Define validation function <#define-validation-function>`__ - `Optimize model using NNCF Post-training Quantization API <#optimize-model-using-nncf-post-training-quantization-api>`__ @@ -69,8 +68,10 @@ The tutorial consists of the following steps: - `Compare Performance of the Original and Quantized Models <#compare-performance-of-the-original-and-quantized-models>`__ -Get Pytorch model ------------------------------------------------------------ +Get Pytorch model +----------------- + + Generally, PyTorch models represent an instance of the `torch.nn.Module `__ @@ -87,8 +88,10 @@ to obtain pre-trained model: In this case, the model creators provide a tool that enables converting the YOLOv7 model to ONNX, so we do not need to do these steps manually. -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + .. code:: ipython3 @@ -119,11 +122,13 @@ Prerequisites .. parsed-literal:: Cloning into 'yolov7'... - remote: Enumerating objects: 1191, done. - remote: Total 1191 (delta 0), reused 0 (delta 0), pack-reused 1191 - Receiving objects: 100% (1191/1191), 74.23 MiB | 3.77 MiB/s, done. - Resolving deltas: 100% (516/516), done. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/226-yolov7-optimization/yolov7 + remote: Enumerating objects: 1197, done. + remote: Counting objects: 100% (6/6), done. + remote: Compressing objects: 100% (5/5), done. + remote: Total 1197 (delta 2), reused 3 (delta 1), pack-reused 1191 + Receiving objects: 100% (1197/1197), 74.23 MiB | 3.54 MiB/s, done. + Resolving deltas: 100% (517/517), done. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/226-yolov7-optimization/yolov7 .. code:: ipython3 @@ -148,12 +153,14 @@ Prerequisites .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/226-yolov7-optimization/yolov7/model/yolov7-tiny.pt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/226-yolov7-optimization/yolov7/model/yolov7-tiny.pt') -Check model inference ---------------------------------------------------------------- +Check model inference +--------------------- + + ``detect.py`` script run pytorch model inference and save image as result, @@ -166,7 +173,7 @@ result, .. parsed-literal:: Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', exist_ok=False, img_size=640, iou_thres=0.45, name='exp', no_trace=False, nosave=False, project='runs/detect', save_conf=False, save_txt=False, source='inference/images/horses.jpg', update=False, view_img=False, weights=['model/yolov7-tiny.pt']) - YOLOR 🚀 v0.1-126-g84932d7 torch 1.13.1+cpu CPU + YOLOR 🚀 v0.1-128-ga207844 torch 1.13.1+cpu CPU Fusing layers... Model Summary: 200 layers, 6219709 parameters, 229245 gradients @@ -174,9 +181,9 @@ result, traced_script_module saved! model is traced! - 5 horses, Done. (73.0ms) Inference, (0.8ms) NMS + 5 horses, Done. (70.2ms) Inference, (0.8ms) NMS The image with the result is saved in: runs/detect/exp/horses.jpg - Done. (0.086s) + Done. (0.083s) .. code:: ipython3 @@ -192,8 +199,10 @@ result, -Export to ONNX --------------------------------------------------------- +Export to ONNX +-------------- + + To export an ONNX format of the model, we will use ``export.py`` script. Let us check its arguments. @@ -270,7 +279,7 @@ an end2end ONNX model, you can check this Import onnx_graphsurgeon failure: No module named 'onnx_graphsurgeon' Namespace(batch_size=1, conf_thres=0.25, device='cpu', dynamic=False, dynamic_batch=False, end2end=False, fp16=False, grid=True, img_size=[640, 640], include_nms=False, int8=False, iou_thres=0.45, max_wh=None, simplify=False, topk_all=100, weights='model/yolov7-tiny.pt') - YOLOR 🚀 v0.1-126-g84932d7 torch 1.13.1+cpu CPU + YOLOR 🚀 v0.1-128-ga207844 torch 1.13.1+cpu CPU Fusing layers... Model Summary: 200 layers, 6219709 parameters, 6219709 gradients @@ -285,16 +294,16 @@ an end2end ONNX model, you can check this Starting ONNX export with onnx 1.15.0... ONNX export success, saved as model/yolov7-tiny.onnx - Export complete (2.48s). Visualize with https://github.com/lutzroeder/netron. + Export complete (2.50s). Visualize with https://github.com/lutzroeder/netron. -Convert ONNX Model to OpenVINO Intermediate Representation (IR) ---------------------------------------------------------------------------------------------------------- +Convert ONNX Model to OpenVINO Intermediate Representation (IR) +--------------------------------------------------------------- -While ONNX models are directly supported by OpenVINO runtime, it can be -useful to convert them to IR format to take the advantage of OpenVINO -model conversion API features. The ``ov.convert_model`` python function -of `model conversion +While ONNX models are directly +supported by OpenVINO runtime, it can be useful to convert them to IR +format to take the advantage of OpenVINO model conversion API features. +The ``ov.convert_model`` python function of `model conversion API `__ can be used for converting the model. The function returns instance of OpenVINO Model class, which is ready to use in Python interface. @@ -309,15 +318,19 @@ However, it can also be save on device in OpenVINO IR format using # serialize model for saving IR ov.save_model(model, 'model/yolov7-tiny.xml') -Verify model inference ----------------------------------------------------------------- +Verify model inference +---------------------- + + To test model work, we create inference pipeline similar to ``detect.py``. The pipeline consists of preprocessing step, inference of OpenVINO model, and results post-processing to get bounding boxes. -Preprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing +~~~~~~~~~~~~~ + + Model input is a tensor with the ``[1, 3, 640, 640]`` shape in ``N, C, H, W`` format, where @@ -405,8 +418,10 @@ To keep specific shape, preprocessing automatically enables padding. COLORS = {name: [np.random.randint(0, 255) for _ in range(3)] for i, name in enumerate(NAMES)} -Postprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Postprocessing +~~~~~~~~~~~~~~ + + Model output contains detection boxes candidates. It is a tensor with the ``[1,25200,85]`` shape in the ``B, N, 85`` format, where: @@ -484,8 +499,10 @@ algorithm and rescale boxes coordinates to original image size. # read converted model model = core.read_model('model/yolov7-tiny.xml') -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -530,11 +547,15 @@ select device from dropdown list for running inference using OpenVINO -Verify model accuracy ---------------------------------------------------------------- +Verify model accuracy +--------------------- + + + +Download dataset +~~~~~~~~~~~~~~~~ + -Download dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ YOLOv7 tiny is pre-trained on the COCO dataset, so in order to evaluate the model accuracy, we need to download it. According to the @@ -576,8 +597,10 @@ the original model evaluation scripts. coco2017labels-segments.zip: 0%| | 0.00/169M [00:00`__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -841,16 +868,64 @@ asymmetric quantization of activations. .. parsed-literal:: - 2023-10-30 23:38:09.707478: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-30 23:38:09.738739: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-06 23:42:22.592237: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-06 23:42:22.623324: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-30 23:38:10.279255: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - Statistics collection: 100%|██████████| 300/300 [00:38<00:00, 7.73it/s] - Applying Fast Bias correction: 100%|██████████| 58/58 [00:04<00:00, 13.46it/s] + 2023-12-06 23:42:23.162794: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + + +.. parsed-literal:: + + Output() + + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:80: RuntimeWarning: invalid value encountered in multiply + return Tensor(self.data * unwrap_tensor_data(other)) + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + +Validate Quantized model inference +---------------------------------- -Validate Quantized model inference ----------------------------------------------------------------------------- .. code:: ipython3 @@ -879,8 +954,10 @@ Validate Quantized model inference -Validate quantized model accuracy ---------------------------------------------------------------------------- +Validate quantized model accuracy +--------------------------------- + + .. code:: ipython3 @@ -906,15 +983,17 @@ Validate quantized model accuracy .. parsed-literal:: Class Images Labels Precision Recall mAP@.5 mAP@.5:.95 - all 5000 36335 0.637 0.508 0.54 0.353 + all 5000 36335 0.634 0.509 0.54 0.353 As we can see, model accuracy slightly changed after quantization. However, if we look at the output image, these changes are not significant. -Compare Performance of the Original and Quantized Models --------------------------------------------------------------------------------------------------- +Compare Performance of the Original and Quantized Models +-------------------------------------------------------- + + Finally, use the OpenVINO `Benchmark Tool `__ @@ -954,18 +1033,18 @@ models. [Step 2/11] Loading OpenVINO Runtime [ WARNING ] Default duration 120 seconds is used for unknown device AUTO [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 13.97 ms + [ INFO ] Read model took 12.83 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: images) : f32 / [...] / [1,3,640,640] @@ -979,7 +1058,7 @@ models. [ INFO ] Model outputs: [ INFO ] output (node: output) : f32 / [...] / [1,25200,85] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 265.32 ms + [ INFO ] Compile model took 265.01 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: torch_jit @@ -1011,17 +1090,17 @@ models. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 45.17 ms + [ INFO ] First inference took 45.58 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 11544 iterations - [ INFO ] Duration: 120105.63 ms + [ INFO ] Count: 11556 iterations + [ INFO ] Duration: 120096.78 ms [ INFO ] Latency: - [ INFO ] Median: 62.16 ms - [ INFO ] Average: 62.28 ms - [ INFO ] Min: 45.96 ms - [ INFO ] Max: 85.36 ms - [ INFO ] Throughput: 96.12 FPS + [ INFO ] Median: 61.89 ms + [ INFO ] Average: 62.22 ms + [ INFO ] Min: 32.61 ms + [ INFO ] Max: 119.36 ms + [ INFO ] Throughput: 96.22 FPS .. code:: ipython3 @@ -1037,18 +1116,18 @@ models. [Step 2/11] Loading OpenVINO Runtime [ WARNING ] Default duration 120 seconds is used for unknown device AUTO [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2023.1.0-12185-9e6b00e51cd-releases/2023/1 + [ INFO ] Build ................................. 2023.2.0-13089-cfd42bd2cb0-HEAD [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 23.91 ms + [ INFO ] Read model took 22.04 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: images) : f32 / [...] / [1,3,640,640] @@ -1062,7 +1141,7 @@ models. [ INFO ] Model outputs: [ INFO ] output (node: output) : f32 / [...] / [1,25200,85] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 456.30 ms + [ INFO ] Compile model took 485.86 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: torch_jit @@ -1094,15 +1173,15 @@ models. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 27.69 ms + [ INFO ] First inference took 26.73 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 32700 iterations - [ INFO ] Duration: 120025.10 ms + [ INFO ] Count: 32994 iterations + [ INFO ] Duration: 120021.82 ms [ INFO ] Latency: - [ INFO ] Median: 21.82 ms - [ INFO ] Average: 21.90 ms - [ INFO ] Min: 16.88 ms - [ INFO ] Max: 44.61 ms - [ INFO ] Throughput: 272.44 FPS + [ INFO ] Median: 21.62 ms + [ INFO ] Average: 21.71 ms + [ INFO ] Min: 16.37 ms + [ INFO ] Max: 44.70 ms + [ INFO ] Throughput: 274.90 FPS diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.jpg b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.jpg index fc548e7ce6e..061ea391916 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.jpg +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1e31df95b9edc90d3ca7f1df06b6a7e752133edf83b803d5b9ef5f85007fb591 -size 64645 +oid sha256:cb7301ac14bde59c43dcdf46dd8c514c27943137b3a2df9d28bfdcd7d35b9ea7 +size 65007 diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.png b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.png index 74fbc2353dc..6f65be57ae8 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.png +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_10_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5fe329db16f68795e1f73c293b698fee6c9c6e4ef606d36357cb06b805de8af2 -size 568876 +oid sha256:e21910f8212b672dab36ea007d4777a725d6dd5772c1493743b6b9fe6c1db68a +size 567956 diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.jpg b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.jpg index 87f3623f3cb..6a936d70f51 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.jpg +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:535860c41070cd771ee02e2983fe7e02a37c4985a2f1f0dabfefb76352db6291 -size 63263 +oid sha256:a1b813c69fdc909df67becf01adc156b5e4f27c83634212e9844fc8610cab20b +size 64089 diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.png b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.png index bd375472d37..8864bcdb7e0 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.png +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_27_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ae19e82dff227f9344c02bd32c02e207e0bb7c0d4e620f8f50013824fa9450c6 -size 574652 +oid sha256:7950e5373569d34d88b5511331eb176e23c16b5ef18b0c6fa803fce3f59fbc4b +size 573726 diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.jpg b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.jpg index 9b8e37417e8..3396a7851fb 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.jpg +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ccacd9e276d177f6f50c7ea6291ae7a1ee68fceace7af1f1a5a953e46be8855d -size 63193 +oid sha256:54f7b6155834878c6410c267849bdf7847a96d481c345a8c562fc3c97d1925b9 +size 64178 diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.png b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.png index 78b7d1275c4..423323f69d5 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.png +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/226-yolov7-optimization-with-output_44_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d8451206e0966b9e5c60310ad856e84cf1ed9f34d59775c6f33bd1bcb7f8fbe0 -size 574615 +oid sha256:68e8f429263c9d8fe420527d299226f25be192f5aa1a3a159562a59b31525eac +size 573483 diff --git a/docs/notebooks/226-yolov7-optimization-with-output_files/index.html b/docs/notebooks/226-yolov7-optimization-with-output_files/index.html index 125d72c1e32..1b1414dfa70 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output_files/index.html +++ b/docs/notebooks/226-yolov7-optimization-with-output_files/index.html @@ -1,12 +1,12 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/226-yolov7-optimization-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/226-yolov7-optimization-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/226-yolov7-optimization-with-output_files/


../
-226-yolov7-optimization-with-output_10_0.jpg       31-Oct-2023 00:35               64645
-226-yolov7-optimization-with-output_10_0.png       31-Oct-2023 00:35              568876
-226-yolov7-optimization-with-output_27_0.jpg       31-Oct-2023 00:35               63263
-226-yolov7-optimization-with-output_27_0.png       31-Oct-2023 00:35              574652
-226-yolov7-optimization-with-output_44_0.jpg       31-Oct-2023 00:35               63193
-226-yolov7-optimization-with-output_44_0.png       31-Oct-2023 00:35              574615
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/226-yolov7-optimization-with-output_files/


../
+226-yolov7-optimization-with-output_10_0.jpg       07-Dec-2023 00:49               65007
+226-yolov7-optimization-with-output_10_0.png       07-Dec-2023 00:49              567956
+226-yolov7-optimization-with-output_27_0.jpg       07-Dec-2023 00:49               64089
+226-yolov7-optimization-with-output_27_0.png       07-Dec-2023 00:49              573726
+226-yolov7-optimization-with-output_44_0.jpg       07-Dec-2023 00:49               64178
+226-yolov7-optimization-with-output_44_0.png       07-Dec-2023 00:49              573483
 

diff --git a/docs/notebooks/227-whisper-convert-with-output.rst b/docs/notebooks/227-whisper-convert-with-output.rst index d420c4b6be0..8998293609e 100644 --- a/docs/notebooks/227-whisper-convert-with-output.rst +++ b/docs/notebooks/227-whisper-convert-with-output.rst @@ -26,6 +26,7 @@ Whisper pipeline with OpenVINO models. **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Instantiate model <#instantiate-model>`__ @@ -54,10 +55,10 @@ Install dependencies. .. code:: ipython3 %pip install -q "openvino>=2023.1.0" - %pip install -q "python-ffmpeg<=1.0.16" moviepy transformers + %pip install -q "python-ffmpeg<=1.0.16" moviepy transformers --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q -I "git+https://github.com/garywu007/pytube.git" %pip install -q -U gradio - %pip install -q -I "git+https://github.com/openai/whisper.git@fcfeaf1b61994c071bba62da47d7846933576ac9" + %pip install -q -I "git+https://github.com/openai/whisper.git@fcfeaf1b61994c071bba62da47d7846933576ac9" --extra-index-url https://download.pytorch.org/whl/cpu Instantiate model ----------------- @@ -563,7 +564,7 @@ Interactive demo .. .. raw:: html -..
+..
.. parsed-literal:: diff --git a/docs/notebooks/229-distilbert-sequence-classification-with-output.rst b/docs/notebooks/229-distilbert-sequence-classification-with-output.rst index 4bc1bb3daf5..a6ee3609e82 100644 --- a/docs/notebooks/229-distilbert-sequence-classification-with-output.rst +++ b/docs/notebooks/229-distilbert-sequence-classification-with-output.rst @@ -20,39 +20,41 @@ sequence classification model using OpenVINO. - `Inference <#inference>`__ - - `For a single input - sentence <#for-a-single-input-sentence>`__ + - `For a single input sentence <#for-a-single-input-sentence>`__ - `Read from a text file <#read-from-a-text-file>`__ -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 - %pip install "openvino>=2023.1.0" transformers + %pip install "openvino>=2023.1.0" transformers --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: - Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2023.1.0) - Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.34.1) - Requirement already satisfied: numpy>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.24.3) - Requirement already satisfied: openvino-telemetry>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2023.2.1) - Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.13.1) - Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.17.3) - Requirement already satisfied: packaging>=20.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (23.2) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.1) - Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2023.10.3) - Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.31.0) - Requirement already satisfied: tokenizers<0.15,>=0.14 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.14.1) - Requirement already satisfied: safetensors>=0.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.0) - Requirement already satisfied: tqdm>=4.27 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (4.66.1) - Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.10.0) - Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.8.0) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.3.1) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.4) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.0.7) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2023.7.22) + Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu + Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2023.2.0) + Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.35.2) + Requirement already satisfied: numpy>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2023.2.1) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.13.1) + Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.19.4) + Requirement already satisfied: packaging>=20.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (23.2) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.1) + Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2023.10.3) + Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.31.0) + Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.15.0) + Requirement already satisfied: safetensors>=0.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.1) + Requirement already satisfied: tqdm>=4.27 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (4.66.1) + Requirement already satisfied: fsspec>=2023.5.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.10.0) + Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.8.0) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.3.2) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.6) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.1.0) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2023.11.17) DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. @@ -66,8 +68,10 @@ Imports import numpy as np import openvino as ov -Initializing the Model ----------------------------------------------------------------- +Initializing the Model +---------------------- + + We will use the transformer-based `DistilBERT base uncased finetuned SST-2 `__ @@ -80,8 +84,10 @@ model from Hugging Face. pretrained_model_name_or_path=checkpoint ) -Initializing the Tokenizer --------------------------------------------------------------------- +Initializing the Tokenizer +-------------------------- + + Text Preprocessing cleans the text-based input data so it can be fed into the model. @@ -100,8 +106,10 @@ understand the context of a sentence. Here, we will use pretrained_model_name_or_path=checkpoint ) -Convert Model to OpenVINO Intermediate Representation format ------------------------------------------------------------------------------------------------------- +Convert Model to OpenVINO Intermediate Representation format +------------------------------------------------------------ + + `Model conversion API `__ @@ -131,7 +139,7 @@ optimal execution on end-point target devices. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:223: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:223: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mask, torch.tensor(torch.finfo(scores.dtype).min) @@ -149,8 +157,10 @@ documentation. `__ - - `Convert model to OpenVINO - IR <#convert-model-to-openvino-ir>`__ + - `Convert model to OpenVINO IR <#convert-model-to-openvino-ir>`__ - `Verify model inference <#verify-model-inference>`__ - `Preprocessing <#preprocessing>`__ - `Postprocessing <#postprocessing>`__ @@ -47,8 +41,7 @@ The tutorial consists of the following steps: - `Download the validation dataset <#download-the-validation-dataset>`__ - - `Define validation - function <#define-validation-function>`__ + - `Define validation function <#define-validation-function>`__ - `Configure Validator helper and create DataLoader <#configure-validator-helper-and-create-dataloader>`__ @@ -66,15 +59,16 @@ The tutorial consists of the following steps: - `Validate quantized model accuracy <#validate-quantized-model-accuracy>`__ -- `Other ways to optimize - model <#other-ways-to-optimize-model>`__ +- `Other ways to optimize model <#other-ways-to-optimize-model>`__ - `Live demo <#live-demo>`__ - `Run Live Object Detection and Segmentation <#run-live-object-detection-and-segmentation>`__ -Get PyTorch model ------------------------------------------------------------ +Get PyTorch model +----------------- + + Generally, PyTorch models represent an instance of the `torch.nn.Module `__ @@ -92,15 +86,17 @@ In this case, the creators of the model provide an API that enables converting the YOLOv8 model to ONNX and then to OpenVINO IR. Therefore, we do not need to do these steps manually. -Prerequisites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Prerequisites +^^^^^^^^^^^^^ + + Install necessary packages. .. code:: ipython3 %pip install -q "openvino>=2023.1.0" "nncf>=2.5.0" - %pip install "ultralytics==8.0.43" onnx + %pip install -q "torch>=2.1" "torchvision>=0.16" "ultralytics==8.0.43" onnx --extra-index-url https://download.pytorch.org/whl/cpu Import required utility functions. The lower cell will download the ``notebook_utils`` Python module from GitHub. @@ -201,8 +197,10 @@ Define utility functions for drawing results -Instantiate model ------------------------------------------------------------ +Instantiate model +----------------- + + For loading the model, required to specify a path to the model checkpoint. It can be some local path or name available on models hub @@ -255,8 +253,10 @@ Let us consider the examples: -Convert model to OpenVINO IR -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert model to OpenVINO IR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + YOLOv8 provides API for convenient model exporting to different formats including OpenVINO IR. ``model.export`` is responsible for model @@ -270,15 +270,19 @@ preserve dynamic shapes in the model. if not seg_model_path.exists(): seg_model.export(format="openvino", dynamic=True, half=False) -Verify model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Verify model inference +~~~~~~~~~~~~~~~~~~~~~~ + + To test model work, we create inference pipeline similar to ``model.predict`` method. The pipeline consists of preprocessing step, inference of OpenVINO model and results post-processing to get results. -Preprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing +~~~~~~~~~~~~~ + + Model input is a tensor with the ``[-1, 3, -1, -1]`` shape in the ``N, C, H, W`` format, where \* ``N`` - number of images in batch (batch @@ -391,8 +395,10 @@ To keep a specific shape, preprocessing automatically enables padding. input_tensor = np.expand_dims(input_tensor, 0) return input_tensor -Postprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Postprocessing +~~~~~~~~~~~~~~ + + The model output contains: - detection boxes candidates - proto mask candidates @@ -488,8 +494,10 @@ batch size - ``C`` - number of candidates - ``H`` - mask height - ``W`` results.append({"det": pred[:, :6].numpy(), "segment": segments}) return results -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + Select device from dropdown list for running inference using OpenVINO @@ -518,8 +526,10 @@ Select device from dropdown list for running inference using OpenVINO -Test on single image -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Test on single image +~~~~~~~~~~~~~~~~~~~~ + + Now, once we have defined preprocessing and postprocessing steps, we are ready to check model prediction. @@ -569,15 +579,19 @@ ready to check model prediction. Great! The result is the same, as produced by original models. -Check model accuracy on the dataset ------------------------------------------------------------------------------ +Check model accuracy on the dataset +----------------------------------- + + For comparing the optimized model result with the original, it is good to know some measurable results in terms of model accuracy on the validation dataset. -Download the validation dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the validation dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + YOLOv8 is pre-trained on the COCO dataset, so to evaluate the model accuracy we need to download it. According to the instructions provided @@ -628,8 +642,10 @@ evaluation function. /home/ea/work/openvino_notebooks/notebooks/230-yolov8-optimization/datasets/coco.yaml: 0%| | 0.00/1… -Define validation function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Define validation function +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -696,8 +712,10 @@ Define validation function pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format print(pf % ('all', total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap)) -Configure Validator helper and create DataLoader -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure Validator helper and create DataLoader +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The original model repository uses a ``Validator`` wrapper, which represents the accuracy validation pipeline. It creates dataloader and @@ -791,8 +809,10 @@ subset difference. *To validate the models on the full dataset set IOU threshold, ``mAP@.5:.95`` - is calculated on range IOU thresholds from 0.5 to 0.95 with step 0.05. -Optimize model using NNCF Post-training Quantization API --------------------------------------------------------------------------------------------------- +Optimize model using NNCF Post-training Quantization API +-------------------------------------------------------- + + `NNCF `__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -914,8 +934,10 @@ point precision, using the ``ignored_scope`` parameter. Quantized segmentation model will be saved to models/yolov8n-seg_openvino_int8_model/yolov8n-seg.xml -Validate Quantized model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate Quantized model inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + ``nncf.quantize`` returns the OpenVINO Model class instance, which is suitable for loading on a device for making predictions. ``INT8`` model @@ -955,13 +977,16 @@ on the image. -Compare the Original and Quantized Models ------------------------------------------------------------------------------------ +Compare the Original and Quantized Models +----------------------------------------- -Compare performance of the Original and Quantized Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Finally, use the OpenVINO `Benchmark + +Compare performance of the Original and Quantized Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Finally, use the OpenVINO +`Benchmark Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. @@ -1158,8 +1183,10 @@ models. [ INFO ] Throughput: 293.15 FPS -Validate quantized model accuracy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate quantized model accuracy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + As we can see, there is no significant difference between ``INT8`` and float model result in a single image test. To understand how @@ -1205,8 +1232,10 @@ accuracy on a dataset. Great! Looks like accuracy was changed, but not significantly and it meets passing criteria. -Other ways to optimize model ----------------------------------------------------------------------- +Other ways to optimize model +---------------------------- + + The performance could be also improved by another OpenVINO method such as async inference pipeline or preprocessing API. @@ -1230,10 +1259,12 @@ utilization. For more information, refer to the overview of tutorial <118-optimize-preprocessing-with-output.html>`__. To see, how it could be used with YOLOV8 object detection model , please, see `Convert and Optimize YOLOv8 real-time object detection with -OpenVINO tutorial <230-yolov8-object-detection-with-output.html>`__ +OpenVINO tutorial <./230-yolov8-object-detection.ipynb>`__ + +Live demo +--------- + -Live demo ---------------------------------------------------- The following code runs model inference on a video: @@ -1339,8 +1370,10 @@ The following code runs model inference on a video: if use_popup: cv2.destroyAllWindows() -Run Live Object Detection and Segmentation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Live Object Detection and Segmentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Use a webcam as the video input. By default, the primary webcam is set with \ ``source=0``. If you have multiple webcams, each one will be diff --git a/docs/notebooks/230-yolov8-instance-segmentation-with-output_files/index.html b/docs/notebooks/230-yolov8-instance-segmentation-with-output_files/index.html index 3c7b8dfd83b..05ae4f77b6c 100644 --- a/docs/notebooks/230-yolov8-instance-segmentation-with-output_files/index.html +++ b/docs/notebooks/230-yolov8-instance-segmentation-with-output_files/index.html @@ -1,13 +1,13 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/230-yolov8-instance-segmentation-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/230-yolov8-instance-segmentation-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/230-yolov8-instance-segmentation-with-output_files/


../
-230-yolov8-instance-segmentation-with-output_11..> 31-Oct-2023 00:35               81079
-230-yolov8-instance-segmentation-with-output_11..> 31-Oct-2023 00:35              790288
-230-yolov8-instance-segmentation-with-output_22..> 31-Oct-2023 00:35              104433
-230-yolov8-instance-segmentation-with-output_22..> 31-Oct-2023 00:35              919053
-230-yolov8-instance-segmentation-with-output_44..> 31-Oct-2023 00:35              103935
-230-yolov8-instance-segmentation-with-output_44..> 31-Oct-2023 00:35              918316
-230-yolov8-instance-segmentation-with-output_60..> 31-Oct-2023 00:35              495438
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/230-yolov8-instance-segmentation-with-output_files/


../
+230-yolov8-instance-segmentation-with-output_11..> 07-Dec-2023 00:49               81079
+230-yolov8-instance-segmentation-with-output_11..> 07-Dec-2023 00:49              790288
+230-yolov8-instance-segmentation-with-output_22..> 07-Dec-2023 00:49              104433
+230-yolov8-instance-segmentation-with-output_22..> 07-Dec-2023 00:49              919053
+230-yolov8-instance-segmentation-with-output_44..> 07-Dec-2023 00:49              103935
+230-yolov8-instance-segmentation-with-output_44..> 07-Dec-2023 00:49              918316
+230-yolov8-instance-segmentation-with-output_60..> 07-Dec-2023 00:49              495438
 

diff --git a/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst b/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst index b2064339cd6..e79775191c5 100644 --- a/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst +++ b/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst @@ -12,17 +12,12 @@ optimize `PyTorch YOLOv8 Pose model `__ with OpenVINO. We consider the steps required for keypoint detection scenario. -The tutorial consists of the following steps: - -- Prepare the PyTorch model. -- Download and prepare a dataset. -- Validate the original model. -- Convert the PyTorch model to OpenVINO IR. -- Validate the converted model. -- Prepare and run optimization pipeline. -- Compare performance of the FP32 and quantized models. -- Compare accuracy of the FP32 and quantized models. -- Live demo +The tutorial consists of the following steps: - Prepare the PyTorch +model. - Download and prepare a dataset. - Validate the original model. +- Convert the PyTorch model to OpenVINO IR. - Validate the converted +model. - Prepare and run optimization pipeline. - Compare performance of +the FP32 and quantized models. - Compare accuracy of the FP32 and +quantized models. - Live demo **Table of contents:** @@ -33,8 +28,7 @@ The tutorial consists of the following steps: - `Instantiate model <#instantiate-model>`__ - - `Convert model to OpenVINO - IR <#convert-model-to-openvino-ir>`__ + - `Convert model to OpenVINO IR <#convert-model-to-openvino-ir>`__ - `Verify model inference <#verify-model-inference>`__ - `Preprocessing <#preprocessing>`__ - `Postprocessing <#postprocessing>`__ @@ -46,8 +40,7 @@ The tutorial consists of the following steps: - `Download the validation dataset <#download-the-validation-dataset>`__ - - `Define validation - function <#define-validation-function>`__ + - `Define validation function <#define-validation-function>`__ - `Configure Validator helper and create DataLoader <#configure-validator-helper-and-create-dataloader>`__ @@ -65,15 +58,16 @@ The tutorial consists of the following steps: - `Compare accuracy of the Original and Quantized Models <#compare-accuracy-of-the-original-and-quantized-models>`__ -- `Other ways to optimize - model <#other-ways-to-optimize-model>`__ +- `Other ways to optimize model <#other-ways-to-optimize-model>`__ - `Live demo <#live-demo>`__ - `Run Keypoint Detection on video <#run-keypoint-detection-on-video>`__ -Get PyTorch model ------------------------------------------------------------ +Get PyTorch model +----------------- + + Generally, PyTorch models represent an instance of the `torch.nn.Module `__ @@ -91,14 +85,16 @@ In this case, the creators of the model provide an API that enables converting the YOLOv8 model to ONNX and then to OpenVINO IR. Therefore, we do not need to do these steps manually. -Prerequisites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Prerequisites +^^^^^^^^^^^^^ + + Install necessary packages. .. code:: ipython3 - %pip install -q "openvino>=2023.1.0" "nncf>=2.5.0" "protobuf==3.20.*" "ultralytics==8.0.159" "onnx" + %pip install -q "openvino>=2023.1.0" "nncf>=2.5.0" "protobuf==3.20.*" "torch>=2.1" "torchvision>=0.16" "ultralytics==8.0.159" "onnx" --extra-index-url https://download.pytorch.org/whl/cpu Import required utility functions. The lower cell will download the ``notebook_utils`` Python module from GitHub. @@ -226,8 +222,10 @@ Define utility functions for drawing results -Instantiate model ------------------------------------------------------------ +Instantiate model +----------------- + + For loading the model, required to specify a path to the model checkpoint. It can be some local path or name available on models hub @@ -271,8 +269,10 @@ Let us consider the examples: -Convert model to OpenVINO IR -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert model to OpenVINO IR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + YOLOv8 provides API for convenient model exporting to different formats including OpenVINO IR. ``model.export`` is responsible for model @@ -286,15 +286,19 @@ preserve dynamic shapes in the model. if not pose_model_path.exists(): pose_model.export(format="openvino", dynamic=True, half=False) -Verify model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Verify model inference +~~~~~~~~~~~~~~~~~~~~~~ + + To test model work, we create inference pipeline similar to ``model.predict`` method. The pipeline consists of preprocessing step, inference of OpenVINO model and results post-processing to get results. -Preprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing +~~~~~~~~~~~~~ + + Model input is a tensor with the ``[-1, 3, -1, -1]`` shape in the ``N, C, H, W`` format, where \* ``N`` - number of images in batch (batch @@ -406,8 +410,10 @@ To keep a specific shape, preprocessing automatically enables padding. input_tensor = np.expand_dims(input_tensor, 0) return input_tensor -Postprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Postprocessing +~~~~~~~~~~~~~~ + + The model output contains detection boxes candidates, it is a tensor with the ``[-1,56,-1]`` shape in the ``B,56,N`` format, where: @@ -480,8 +486,10 @@ After prediction detection box has the [``x``, ``y``, ``h``, ``w``, return results -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + Select device from dropdown list for running inference using OpenVINO @@ -510,8 +518,10 @@ Select device from dropdown list for running inference using OpenVINO -Test on single image -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Test on single image +~~~~~~~~~~~~~~~~~~~~ + + Now, once we have defined preprocessing and postprocessing steps, we are ready to check model prediction. @@ -558,15 +568,19 @@ ready to check model prediction. Great! The result is the same, as produced by original models. -Check model accuracy on the dataset ------------------------------------------------------------------------------ +Check model accuracy on the dataset +----------------------------------- + + For comparing the optimized model result with the original, it is good to know some measurable results in terms of model accuracy on the validation dataset. -Download the validation dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the validation dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + YOLOv8 is pre-trained on the COCO dataset, so to evaluate the model accuracy we need to download it. According to the instructions provided @@ -615,8 +629,10 @@ evaluation function. datasets/coco-pose.yaml: 0%| | 0.00/781 [00:00`__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -906,8 +926,10 @@ point precision, using the ``ignored_scope`` parameter. Quantized keypoint detection model will be saved to models/yolov8n-pose_openvino_int8_model/yolov8n-pose.xml -Validate Quantized model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate Quantized model inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + ``nncf.quantize`` returns the OpenVINO Model class instance, which is suitable for loading on a device for making predictions. ``INT8`` model @@ -947,13 +969,16 @@ on the image. -Compare the Original and Quantized Models ------------------------------------------------------------------------------------ +Compare the Original and Quantized Models +----------------------------------------- -Compare performance of the Original and Quantized Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Finally, use the OpenVINO `Benchmark + +Compare performance of the Original and Quantized Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Finally, use the OpenVINO +`Benchmark Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. @@ -1149,8 +1174,10 @@ models. [ INFO ] Throughput: 426.35 FPS -Compare accuracy of the Original and Quantized Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Compare accuracy of the Original and Quantized Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + As we can see, there is no significant difference between ``INT8`` and float model result in a single image test. To understand how @@ -1192,8 +1219,10 @@ accuracy on a dataset. Great! Looks like accuracy was changed, but not significantly and it meets passing criteria. -Other ways to optimize model ----------------------------------------------------------------------- +Other ways to optimize model +---------------------------- + + The performance could be also improved by another OpenVINO method such as async inference pipeline or preprocessing API. @@ -1217,10 +1246,12 @@ utilization. For more information, refer to the overview of tutorial <118-optimize-preprocessing-with-output.html>`__. To see, how it could be used with YOLOV8 object detection model , please, see `Convert and Optimize YOLOv8 real-time object detection with -OpenVINO tutorial <230-yolov8-object-detection-with-output.html>`__ +OpenVINO tutorial <./230-yolov8-object-detection.ipynb>`__ + +Live demo +--------- + -Live demo ---------------------------------------------------- The following code runs model inference on a video: @@ -1326,8 +1357,10 @@ The following code runs model inference on a video: if use_popup: cv2.destroyAllWindows() -Run Keypoint Detection on video -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Keypoint Detection on video +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 diff --git a/docs/notebooks/230-yolov8-keypoint-detection-with-output_files/index.html b/docs/notebooks/230-yolov8-keypoint-detection-with-output_files/index.html index f6787514b14..ec5f0b474d3 100644 --- a/docs/notebooks/230-yolov8-keypoint-detection-with-output_files/index.html +++ b/docs/notebooks/230-yolov8-keypoint-detection-with-output_files/index.html @@ -1,13 +1,13 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/230-yolov8-keypoint-detection-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/230-yolov8-keypoint-detection-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/230-yolov8-keypoint-detection-with-output_files/


../
-230-yolov8-keypoint-detection-with-output_11_1.jpg 31-Oct-2023 00:35               58622
-230-yolov8-keypoint-detection-with-output_11_1.png 31-Oct-2023 00:35              581068
-230-yolov8-keypoint-detection-with-output_22_0.jpg 31-Oct-2023 00:35               58280
-230-yolov8-keypoint-detection-with-output_22_0.png 31-Oct-2023 00:35              584143
-230-yolov8-keypoint-detection-with-output_46_0.jpg 31-Oct-2023 00:35               58058
-230-yolov8-keypoint-detection-with-output_46_0.png 31-Oct-2023 00:35              584051
-230-yolov8-keypoint-detection-with-output_62_0.png 31-Oct-2023 00:35              490879
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/230-yolov8-keypoint-detection-with-output_files/


../
+230-yolov8-keypoint-detection-with-output_11_1.jpg 07-Dec-2023 00:49               58622
+230-yolov8-keypoint-detection-with-output_11_1.png 07-Dec-2023 00:49              581068
+230-yolov8-keypoint-detection-with-output_22_0.jpg 07-Dec-2023 00:49               58280
+230-yolov8-keypoint-detection-with-output_22_0.png 07-Dec-2023 00:49              584143
+230-yolov8-keypoint-detection-with-output_46_0.jpg 07-Dec-2023 00:49               58058
+230-yolov8-keypoint-detection-with-output_46_0.png 07-Dec-2023 00:49              584051
+230-yolov8-keypoint-detection-with-output_62_0.png 07-Dec-2023 00:49              490879
 

diff --git a/docs/notebooks/230-yolov8-object-detection-with-output.rst b/docs/notebooks/230-yolov8-object-detection-with-output.rst index 8a7e28c2083..d45e7108995 100644 --- a/docs/notebooks/230-yolov8-object-detection-with-output.rst +++ b/docs/notebooks/230-yolov8-object-detection-with-output.rst @@ -10,18 +10,13 @@ This tutorial demonstrates step-by-step instructions on how to run and optimize PyTorch YOLOv8 with OpenVINO. We consider the steps required for object detection scenario. -The tutorial consists of the following steps: - -- Prepare the PyTorch model. -- Download and prepare a dataset. -- Validate the original model. -- Convert the PyTorch model to OpenVINO IR. -- Validate the converted model. -- Prepare and run optimization pipeline. -- Compare performance ofthe FP32 and quantized models. -- Compare accuracy of the FP32 and quantized models. -- Other optimization possibilities with OpenVINO api -- Live demo +The tutorial consists of the following steps: - Prepare the PyTorch +model. - Download and prepare a dataset. - Validate the original model. +- Convert the PyTorch model to OpenVINO IR. - Validate the converted +model. - Prepare and run optimization pipeline. - Compare performance of +the FP32 and quantized models. - Compare accuracy of the FP32 and +quantized models. - Other optimization possibilities with OpenVINO api - +Live demo **Table of contents:** @@ -32,8 +27,7 @@ The tutorial consists of the following steps: - `Instantiate model <#instantiate-model>`__ - - `Convert model to OpenVINO - IR <#convert-model-to-openvino-ir>`__ + - `Convert model to OpenVINO IR <#convert-model-to-openvino-ir>`__ - `Verify model inference <#verify-model-inference>`__ - `Preprocessing <#preprocessing>`__ - `Postprocessing <#postprocessing>`__ @@ -45,8 +39,7 @@ The tutorial consists of the following steps: - `Download the validation dataset <#download-the-validation-dataset>`__ - - `Define validation - function <#define-validation-function>`__ + - `Define validation function <#define-validation-function>`__ - `Configure Validator helper and create DataLoader <#configure-validator-helper-and-create-dataloader>`__ @@ -72,8 +65,7 @@ The tutorial consists of the following steps: - `Initialize PrePostProcessing API <#initialize-prepostprocessing-api>`__ - - `Define input data - format <#define-input-data-format>`__ + - `Define input data format <#define-input-data-format>`__ - `Describe preprocessing steps <#describe-preprocessing-steps>`__ - `Integrating Steps into a @@ -83,8 +75,10 @@ The tutorial consists of the following steps: - `Run Live Object Detection <#run-live-object-detection>`__ -Get PyTorch model ------------------------------------------------------------ +Get PyTorch model +----------------- + + Generally, PyTorch models represent an instance of the `torch.nn.Module `__ @@ -102,15 +96,17 @@ In this case, the creators of the model provide an API that enables converting the YOLOv8 model to ONNX and then to OpenVINO IR. Therefore, we do not need to do these steps manually. -Prerequisites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Prerequisites +^^^^^^^^^^^^^ + + Install necessary packages. .. code:: ipython3 %pip install -q "openvino>=2023.1.0" "nncf>=2.5.0" - %pip install -q "ultralytics==8.0.43" onnx + %pip install -q "torch>=2.1" "torchvision>=0.16" "ultralytics==8.0.43" onnx --extra-index-url https://download.pytorch.org/whl/cpu Import required utility functions. The lower cell will download the ``notebook_utils`` Python module from GitHub. @@ -205,8 +201,10 @@ Define utility functions for drawing results -Instantiate model ------------------------------------------------------------ +Instantiate model +----------------- + + There are `several models `__ available in the @@ -260,8 +258,10 @@ Let us consider the examples: -Convert model to OpenVINO IR -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert model to OpenVINO IR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + YOLOv8 provides API for convenient model exporting to different formats including OpenVINO IR. ``model.export`` is responsible for model @@ -275,15 +275,19 @@ preserve dynamic shapes in the model. if not det_model_path.exists(): det_model.export(format="openvino", dynamic=True, half=False) -Verify model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Verify model inference +~~~~~~~~~~~~~~~~~~~~~~ + + To test model work, we create inference pipeline similar to ``model.predict`` method. The pipeline consists of preprocessing step, inference of OpenVINO model and results post-processing to get results. -Preprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing +~~~~~~~~~~~~~ + + Model input is a tensor with the ``[-1, 3, -1, -1]`` shape in the ``N, C, H, W`` format, where \* ``N`` - number of images in batch (batch @@ -396,8 +400,10 @@ To keep a specific shape, preprocessing automatically enables padding. input_tensor = np.expand_dims(input_tensor, 0) return input_tensor -Postprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Postprocessing +~~~~~~~~~~~~~~ + + The model output contains detection boxes candidates, it is a tensor with the ``[-1,84,-1]`` shape in the ``B,84,N`` format, where: @@ -461,8 +467,10 @@ Finally, detection box has the [``x``, ``y``, ``h``, ``w``, return results -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + Select device from dropdown list for running inference using OpenVINO @@ -491,8 +499,10 @@ Select device from dropdown list for running inference using OpenVINO -Test on single image -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Test on single image +~~~~~~~~~~~~~~~~~~~~ + + Now, once we have defined preprocessing and postprocessing steps, we are ready to check model prediction for object detection. @@ -537,15 +547,19 @@ ready to check model prediction for object detection. -Check model accuracy on the dataset ------------------------------------------------------------------------------ +Check model accuracy on the dataset +----------------------------------- + + For comparing the optimized model result with the original, it is good to know some measurable results in terms of model accuracy on the validation dataset. -Download the validation dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the validation dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + YOLOv8 is pre-trained on the COCO dataset, so to evaluate the model accuracy we need to download it. According to the instructions provided @@ -594,8 +608,10 @@ evaluation function. datasets/coco.yaml: 0%| | 0.00/1.25k [00:00`__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -875,8 +895,10 @@ point precision, using the ``ignored_scope`` parameter. Quantized detection model will be saved to models/yolov8n_openvino_int8_model/yolov8n.xml -Validate Quantized model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate Quantized model inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + ``nncf.quantize`` returns the OpenVINO Model class instance, which is suitable for loading on a device for making predictions. ``INT8`` model @@ -916,11 +938,15 @@ on the image. -Compare the Original and Quantized Models ------------------------------------------------------------------------------------ +Compare the Original and Quantized Models +----------------------------------------- + + + +Compare performance object detection models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + -Compare performance object detection models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Finally, use the OpenVINO `Benchmark Tool `__ @@ -1118,8 +1144,10 @@ models. [ INFO ] Throughput: 208.39 FPS -Validate quantized model accuracy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate quantized model accuracy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + As we can see, there is no significant difference between ``INT8`` and float model result in a single image test. To understand how @@ -1161,24 +1189,28 @@ accuracy on a dataset. Great! Looks like accuracy was changed, but not significantly and it meets passing criteria. -Next steps ----------------------------------------------------- +Next steps +---------- -This section contains suggestions on how to additionally improve the -performance of your application using OpenVINO. +This section contains +suggestions on how to additionally improve the performance of your +application using OpenVINO. -Async inference pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Async inference pipeline +~~~~~~~~~~~~~~~~~~~~~~~~ -The key advantage of the Async API is that when a device is busy with -inference, the application can perform other tasks in parallel (for -example, populating inputs or scheduling other requests) rather than -wait for the current inference to complete first. To understand how to -perform async inference using openvino, refer to `Async API +The key advantage of the Async +API is that when a device is busy with inference, the application can +perform other tasks in parallel (for example, populating inputs or +scheduling other requests) rather than wait for the current inference to +complete first. To understand how to perform async inference using +openvino, refer to `Async API tutorial <115-async-api-with-output.html>`__ -Integration preprocessing to model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Integration preprocessing to model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Preprocessing API enables making preprocessing a part of the model reducing application code and dependency on additional image processing @@ -1198,8 +1230,10 @@ The integration process consists of the following steps: 1. Initialize a PrePostProcessing object. 2. Define the input data format. 3. Describe preprocessing steps. 4. Integrating Steps into a Model. -Initialize PrePostProcessing API -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Initialize PrePostProcessing API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + The ``openvino.preprocess.PrePostProcessor`` class enables specifying preprocessing and postprocessing steps for a model. @@ -1210,8 +1244,10 @@ preprocessing and postprocessing steps for a model. ppp = PrePostProcessor(quantized_det_model) -Define input data format -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Define input data format +^^^^^^^^^^^^^^^^^^^^^^^^ + + To address particular input of a model/preprocessor, the ``input(input_id)`` method, where ``input_id`` is a positional index or @@ -1229,8 +1265,10 @@ provide this to the tensor description. To perform layout conversion, we also should provide information about layout expected by model -Describe preprocessing steps -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Describe preprocessing steps +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Our preprocessing function contains the following steps: \* Convert the data type from ``U8`` to ``FP32``. \* Convert the data layout from @@ -1259,8 +1297,10 @@ preprocessing steps: -Integrating Steps into a Model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Integrating Steps into a Model +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Once the preprocessing steps have been finished, the model can be finally built. Additionally, we can save a completed model to OpenVINO @@ -1308,8 +1348,10 @@ device. Now, we can skip these preprocessing steps in detect function: -Live demo ---------------------------------------------------- +Live demo +--------- + + The following code runs model inference on a video: @@ -1416,8 +1458,10 @@ The following code runs model inference on a video: if use_popup: cv2.destroyAllWindows() -Run Live Object Detection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Live Object Detection +~~~~~~~~~~~~~~~~~~~~~~~~~ + + Use a webcam as the video input. By default, the primary webcam is set with \ ``source=0``. If you have multiple webcams, each one will be diff --git a/docs/notebooks/230-yolov8-object-detection-with-output_files/230-yolov8-object-detection-with-output_22_0.jpg b/docs/notebooks/230-yolov8-object-detection-with-output_files/230-yolov8-object-detection-with-output_22_0.jpg deleted file mode 100644 index 54446c32461..00000000000 --- a/docs/notebooks/230-yolov8-object-detection-with-output_files/230-yolov8-object-detection-with-output_22_0.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a33a9db3664a7c515e8fe0b4cbb3d76ffd5caa1aae461c9a2f59de83337a4b69 -size 110022 diff --git a/docs/notebooks/230-yolov8-object-detection-with-output_files/index.html b/docs/notebooks/230-yolov8-object-detection-with-output_files/index.html index 7caea3730df..7b6abb15729 100644 --- a/docs/notebooks/230-yolov8-object-detection-with-output_files/index.html +++ b/docs/notebooks/230-yolov8-object-detection-with-output_files/index.html @@ -1,15 +1,15 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/230-yolov8-object-detection-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/230-yolov8-object-detection-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/230-yolov8-object-detection-with-output_files/


../
-230-yolov8-object-detection-with-output_11_1.jpg   31-Oct-2023 00:35              110998
-230-yolov8-object-detection-with-output_11_1.png   31-Oct-2023 00:35              910454
-230-yolov8-object-detection-with-output_22_0.jpg   31-Oct-2023 00:35              110022
-230-yolov8-object-detection-with-output_22_0.png   31-Oct-2023 00:35              929067
-230-yolov8-object-detection-with-output_45_0.jpg   31-Oct-2023 00:35              110900
-230-yolov8-object-detection-with-output_45_0.png   31-Oct-2023 00:35              929911
-230-yolov8-object-detection-with-output_68_0.jpg   31-Oct-2023 00:35              110900
-230-yolov8-object-detection-with-output_68_0.png   31-Oct-2023 00:35              929911
-230-yolov8-object-detection-with-output_74_0.png   31-Oct-2023 00:35              491904
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/230-yolov8-object-detection-with-output_files/


../
+230-yolov8-object-detection-with-output_11_1.jpg   07-Dec-2023 00:49              110998
+230-yolov8-object-detection-with-output_11_1.png   07-Dec-2023 00:49              910454
+230-yolov8-object-detection-with-output_22_0.jpg   07-Dec-2023 00:49              110022
+230-yolov8-object-detection-with-output_22_0.png   07-Dec-2023 00:49              929067
+230-yolov8-object-detection-with-output_45_0.jpg   07-Dec-2023 00:49              110900
+230-yolov8-object-detection-with-output_45_0.png   07-Dec-2023 00:49              929911
+230-yolov8-object-detection-with-output_68_0.jpg   07-Dec-2023 00:49              110900
+230-yolov8-object-detection-with-output_68_0.png   07-Dec-2023 00:49              929911
+230-yolov8-object-detection-with-output_74_0.png   07-Dec-2023 00:49              491904
 

diff --git a/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst b/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst index f57eaa0c65b..bdc9b80be3a 100644 --- a/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst +++ b/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst @@ -34,10 +34,8 @@ Notebook contains the following steps: - `Prerequisites <#prerequisites>`__ -- `Create Pytorch Models - pipeline <#create-pytorch-models-pipeline>`__ -- `Convert Models to OpenVINO - IR <#convert-models-to-openvino-ir>`__ +- `Create Pytorch Models pipeline <#create-pytorch-models-pipeline>`__ +- `Convert Models to OpenVINO IR <#convert-models-to-openvino-ir>`__ - `Text Encoder <#text-encoder>`__ - `VAE <#vae>`__ @@ -46,27 +44,29 @@ Notebook contains the following steps: - `Prepare Inference Pipeline <#prepare-inference-pipeline>`__ - `Quantization <#quantization>`__ - - `Prepare calibration - dataset <#prepare-calibration-dataset>`__ + - `Prepare calibration dataset <#prepare-calibration-dataset>`__ - `Run quantization <#run-quantization>`__ - `Compare inference time of the FP16 and INT8 models <#compare-inference-time-of-the-fp-and-int-models>`__ -- `Interactive demo with - Gradio <#interactive-demo-with-gradio>`__ +- `Interactive demo with Gradio <#interactive-demo-with-gradio>`__ + +Prerequisites +------------- + -Prerequisites -------------------------------------------------------- Install necessary packages .. code:: ipython3 - %pip install -q "transformers>=4.25.1" accelerate gradio datasets diffusers + %pip install -q "transformers>=4.25.1" accelerate gradio datasets diffusers --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "openvino>=2023.1.0" -Create Pytorch Models pipeline ------------------------------------------------------------------------- +Create Pytorch Models pipeline +------------------------------ + + ``StableDiffusionInstructPix2PixPipeline`` is an end-to-end inference pipeline that you can use to edit images from text instructions with @@ -95,11 +95,13 @@ First, we load the pre-trained weights of all components of the model. del pipe -Convert Models to OpenVINO IR ------------------------------------------------------------------------ +Convert Models to OpenVINO IR +----------------------------- + + OpenVINO supports PyTorch models using `Model Conversion -API `__ +API `__ to convert the model to IR format. ``ov.convert_model`` function accepts PyTorch model object and example input and then converts it to ``ov.Model`` class instance that ready to use for loading on device or @@ -121,8 +123,10 @@ The model consists of three important parts: Let us convert each part. -Text Encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text Encoder +~~~~~~~~~~~~ + + The text-encoder is responsible for transforming the input prompt, for example, “a photo of an astronaut riding a horse” into an embedding @@ -202,8 +206,10 @@ hidden states. -VAE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +VAE +~~~ + + The VAE model consists of two parts: an encoder and a decoder. @@ -313,8 +319,10 @@ into two independent models. -Unet -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Unet +~~~~ + + The Unet model has three inputs: @@ -391,8 +399,10 @@ Model predicts the ``sample`` state for the next step. -Prepare Inference Pipeline --------------------------------------------------------------------- +Prepare Inference Pipeline +-------------------------- + + Putting it all together, let us now take a closer look at how the model inference works by illustrating the logical flow. @@ -998,8 +1008,10 @@ generation. Nice. As you can see, the picture has quite a high definition 🔥. -Quantization -------------------------------------------------------- +Quantization +------------ + + `NNCF `__ enables post-training quantization by adding quantization layers into model @@ -1052,8 +1064,10 @@ Let’s load ``skip magic`` extension to skip quantization if %load_ext skip_kernel_extension -Prepare calibration dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare calibration dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + We use a portion of `fusing/instructpix2pix-1000-samples `__ @@ -1127,8 +1141,10 @@ model inputs for calibration we should customize ``CompiledModel``. 0%| | 0/300 [00:00 -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/231-instruct-pix2pix-image-editing-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/231-instruct-pix2pix-image-editing-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/231-instruct-pix2pix-image-editing-with-output_files/


../
-231-instruct-pix2pix-image-editing-with-output_..> 31-Oct-2023 00:35             3699096
-231-instruct-pix2pix-image-editing-with-output_..> 31-Oct-2023 00:35             3646410
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/231-instruct-pix2pix-image-editing-with-output_files/


../
+231-instruct-pix2pix-image-editing-with-output_..> 07-Dec-2023 00:49             3699096
+231-instruct-pix2pix-image-editing-with-output_..> 07-Dec-2023 00:49             3646410
 

diff --git a/docs/notebooks/233-blip-convert-with-output.rst b/docs/notebooks/233-blip-convert-with-output.rst index a2265cfed79..e4a3d8e7fac 100644 --- a/docs/notebooks/233-blip-convert-with-output.rst +++ b/docs/notebooks/233-blip-convert-with-output.rst @@ -31,8 +31,7 @@ The tutorial consists of the following parts: - `Visual Question Answering <#visual-question-answering>`__ - `Instantiate Model <#instantiate-model>`__ -- `Convert Models to OpenVINO - IR <#convert-models-to-openvino-ir>`__ +- `Convert Models to OpenVINO IR <#convert-models-to-openvino-ir>`__ - `Vision Model <#vision-model>`__ - `Text Encoder <#text-encoder>`__ @@ -40,8 +39,7 @@ The tutorial consists of the following parts: - `Run OpenVINO Model <#run-openvino-model>`__ - - `Prepare Inference - Pipeline <#prepare-inference-pipeline>`__ + - `Prepare Inference Pipeline <#prepare-inference-pipeline>`__ - `Select inference device <#select-inference-device>`__ - `Image Captioning <#image-captioning>`__ - `Question Answering <#question-answering>`__ @@ -49,8 +47,10 @@ The tutorial consists of the following parts: - `Interactive demo <#interactive-demo>`__ - `Next steps <#next-steps>`__ -Background ----------------------------------------------------- +Background +---------- + + Visual language processing is a branch of artificial intelligence that focuses on creating algorithms designed to enable computers to more @@ -79,8 +79,10 @@ context are variables requested by a user. This notebook does not focus on Text to Image retrieval. Instead, it considers Image Captioning and Visual Question Answering. -Image Captioning -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image Captioning +~~~~~~~~~~~~~~~~ + + Image Captioning is the task of describing the content of an image in words. This task lies at the intersection of computer vision and natural @@ -91,8 +93,10 @@ decoded into a descriptive text sequence. |image1| -Visual Question Answering -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Visual Question Answering +~~~~~~~~~~~~~~~~~~~~~~~~~ + + Visual Question Answering (VQA) is the task of answering text-based questions about image content. @@ -189,8 +193,10 @@ There are a lot of applications for visual question answering: .. |image4| image:: https://user-images.githubusercontent.com/29454499/222094861-3cafdf9f-d700-4741-b6c5-fb09c1a4da9a.png .. |image5| image:: https://user-images.githubusercontent.com/29454499/222095118-3d5826e4-2662-4d1c-abf2-a515f23d6d6a.png -Instantiate Model ------------------------------------------------------------ +Instantiate Model +----------------- + + The BLIP model was proposed in the `BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and @@ -239,8 +245,7 @@ text and vision modalities and postprocessing of generation results. .. code:: ipython3 - %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision - %pip install -q "transformers >= 4.26.0" gradio "openvino>=2023.1.0" matplotlib + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision "transformers>=4.26.0" gradio "openvino>=2023.1.0" matplotlib .. code:: ipython3 @@ -315,8 +320,10 @@ text and vision modalities and postprocessing of generation results. .. image:: 233-blip-convert-with-output_files/233-blip-convert-with-output_7_0.png -Convert Models to OpenVINO IR ------------------------------------------------------------------------ +Convert Models to OpenVINO IR +----------------------------- + + Starting from OpenVINO 2023.0 release, OpenVINO supports direct PyTorch models conversion to OpenVINO Intermediate Representation (IR) format to @@ -337,8 +344,10 @@ The model consists of three parts: To be able to perform multiple tasks, using the same model components, you should convert each part independently. -Vision Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Vision Model +~~~~~~~~~~~~ + + The vision model accepts float input tensors with the [1,3,384,384] shape, containing RGB image pixel values normalized in the [0,1] range. @@ -375,8 +384,10 @@ shape, containing RGB image pixel values normalized in the [0,1] range. Vision model will be loaded from blip_vision_model.xml -Text Encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text Encoder +~~~~~~~~~~~~ + + The text encoder is used by visual question answering tasks to build a question embedding representation. It takes ``input_ids`` with a @@ -412,8 +423,10 @@ model and attention masks for them. Text encoder will be loaded from blip_text_encoder.xml -Text Decoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text Decoder +~~~~~~~~~~~~ + + The text decoder is responsible for generating the sequence of tokens to represent model output (answer to question or caption), using an image @@ -493,17 +506,21 @@ shapes. Text decoder will be loaded from blip_text_decoder_with_past.xml -Run OpenVINO Model ------------------------------------------------------------- +Run OpenVINO Model +------------------ + + + +Prepare Inference Pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~ + -Prepare Inference Pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ As discussed before, the model consists of several blocks which can be reused for building pipelines for different tasks. In the diagram below, you can see how image captioning works: -|image21| +|image01| The visual model accepts the image preprocessed by ``BlipProcessor`` as input and produces image embeddings, which are directly passed to the @@ -517,20 +534,22 @@ tokenized by ``BlipProcessor`` are provided to the text encoder and then multimodal question embedding is passed to the text decoder for performing generation of answers. -|image31| +|image11| The next step is implementing both pipelines using OpenVINO models. -.. |image21| image:: https://user-images.githubusercontent.com/29454499/221865836-a56da06e-196d-449c-a5dc-4136da6ab5d5.png -.. |image31| image:: https://user-images.githubusercontent.com/29454499/221868167-d0081add-d9f3-4591-80e7-4753c88c1d0a.png +.. |image01| image:: https://user-images.githubusercontent.com/29454499/221865836-a56da06e-196d-449c-a5dc-4136da6ab5d5.png +.. |image11| image:: https://user-images.githubusercontent.com/29454499/221868167-d0081add-d9f3-4591-80e7-4753c88c1d0a.png .. code:: ipython3 # create OpenVINO Core object instance core = ov.Core() -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -586,8 +605,10 @@ initial token for decoder work. Now, the model is ready for generation. -Image Captioning -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image Captioning +~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -600,8 +621,10 @@ Image Captioning .. image:: 233-blip-convert-with-output_files/233-blip-convert-with-output_25_0.png -Question Answering -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Question Answering +~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -626,8 +649,10 @@ Question Answering Processing time: 0.1617 -Interactive demo ----------------------------------------------------------- +Interactive demo +---------------- + + .. code:: ipython3 @@ -671,10 +696,12 @@ Interactive demo # demo.launch(server_name='your server name', server_port='server port in int') # Read more in the docs: https://gradio.app/docs/ -Next steps ----------------------------------------------------- +Next steps +---------- -Open the `233-blip-optimize <233-blip-optimize-with-output.html>`__ notebook to + + +Open the `233-blip-optimize <233-blip-optimize.ipynb>`__ notebook to quantize vision and text encoder models with the Post-training Quantization API of NNCF and compress weights of the text decoder. Then compare the converted and optimized OpenVINO models. diff --git a/docs/notebooks/233-blip-convert-with-output_files/index.html b/docs/notebooks/233-blip-convert-with-output_files/index.html index 1bbb317e912..d5bd5b3d312 100644 --- a/docs/notebooks/233-blip-convert-with-output_files/index.html +++ b/docs/notebooks/233-blip-convert-with-output_files/index.html @@ -1,9 +1,9 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/233-blip-convert-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/233-blip-convert-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/233-blip-convert-with-output_files/


../
-233-blip-convert-with-output_25_0.png              31-Oct-2023 00:35              206940
-233-blip-convert-with-output_27_0.png              31-Oct-2023 00:35              210551
-233-blip-convert-with-output_7_0.png               31-Oct-2023 00:35              210551
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/233-blip-convert-with-output_files/


../
+233-blip-convert-with-output_25_0.png              07-Dec-2023 00:49              206940
+233-blip-convert-with-output_27_0.png              07-Dec-2023 00:49              210551
+233-blip-convert-with-output_7_0.png               07-Dec-2023 00:49              210551
 

diff --git a/docs/notebooks/234-encodec-audio-compression-with-output.rst b/docs/notebooks/234-encodec-audio-compression-with-output.rst index 419ccc2cbe1..217a132bd68 100644 --- a/docs/notebooks/234-encodec-audio-compression-with-output.rst +++ b/docs/notebooks/234-encodec-audio-compression-with-output.rst @@ -47,8 +47,10 @@ and original `repo `__. - `Run EnCodec with OpenVINO <#run-encodec-with-openvino>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + Install required dependencies: @@ -59,13 +61,17 @@ Install required dependencies: .. parsed-literal:: - DEPRECATION: git+https://\*\*\*\*@github.com/eaidova/encodec#egg=encodec;python_version=="3.7" contains an egg fragment with a non-PEP 508 name pip 25.0 will enforce this behaviour change. A possible replacement is to use the req @ url syntax, and remove the egg fragment. Discussion can be found at https://github.com/pypa/pip/issues/11617 - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.\*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + pyannote-audio 2.0.1 requires torchaudio<1.0,>=0.10, but you have torchaudio 2.1.1+cpu which is incompatible. + torchvision 0.14.1+cpu requires torch==1.13.1, but you have torch 2.1.1+cpu which is incompatible. Note: you may need to restart the kernel to use updated packages. -Instantiate audio compression pipeline --------------------------------------------------------------------------------- +Instantiate audio compression pipeline +-------------------------------------- + + `Codecs `__, which act as encoders and decoders for streams of data, help empower most of the audio @@ -122,8 +128,17 @@ bandwidth. model = EncodecModel.encodec_model_24khz() model.set_target_bandwidth(6.0) -Explore EnCodec pipeline ------------------------------------------------------------------- + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") + + +Explore EnCodec pipeline +------------------------ + + Let us explore model capabilities on example audio: @@ -173,8 +188,10 @@ Let us explore model capabilities on example audio: .. image:: 234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_6_2.png -Preprocessing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing +~~~~~~~~~~~~~ + + To achieve the best result, audio should have the number of channels and sample rate expected by the model. If audio does not fulfill these @@ -201,8 +218,10 @@ number of channels using the ``convert_audio`` function. wav = convert_audio(wav, sr, model_sr, model_channels) -Encoding -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Encoding +~~~~~~~~ + + Audio waveform should be split by chunks and then encoded by Encoder model, then compressed by quantizer for reducing memory. The result of @@ -250,8 +269,10 @@ Let us compare obtained compression result: Great! Now, we see the power of hyper compression. Binary size of a file becomes 60 times smaller and more suitable for sending via network. -Decompression -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Decompression +~~~~~~~~~~~~~ + + After successful sending of the compressed audio, it should be decompressed on the recipient’s side. The decoder model is responsible @@ -262,6 +283,13 @@ similar as possible to the original. out, out_sr = decompress(out_file.read_bytes()) + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") + + .. code:: ipython3 output_file = "decopressed.wav" @@ -286,7 +314,7 @@ audio. @@ -299,8 +327,10 @@ audio. Nice! Audio sounds close to original. -Convert model to OpenVINO Intermediate Representation format ------------------------------------------------------------------------------------------------------- +Convert model to OpenVINO Intermediate Representation format +------------------------------------------------------------ + + For best results with OpenVINO, it is recommended to convert the model to OpenVINO IR format. OpenVINO supports PyTorch via ONNX conversion. We @@ -358,25 +388,19 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! max_pad = max(padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if length <= max_pad: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/symbolic_opset9.py:4315: UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable length with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/symbolic_opset9.py:4662: UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable length with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model. warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) - _C._jit_pass_onnx_node_shape_type_inference(node, params_dict, opset_version) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: Constant folding - Only steps=1 can be constant folded for opset >= 10 onnx::Slice op. Constant folding not applied. (Triggered internally at ../torch/csrc/jit/passes/onnx/constant_fold.cpp:179.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:702: UserWarning: Constant folding - Only steps=1 can be constant folded for opset >= 10 onnx::Slice op. Constant folding not applied. (Triggered internally at ../torch/csrc/jit/passes/onnx/constant_fold.cpp:179.) _C._jit_pass_onnx_graph_shape_type_inference( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) - _C._jit_pass_onnx_graph_shape_type_inference( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: Constant folding - Only steps=1 can be constant folded for opset >= 10 onnx::Slice op. Constant folding not applied. (Triggered internally at ../torch/csrc/jit/passes/onnx/constant_fold.cpp:179.) - _C._jit_pass_onnx_graph_shape_type_inference( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1209: UserWarning: Constant folding - Only steps=1 can be constant folded for opset >= 10 onnx::Slice op. Constant folding not applied. (Triggered internally at ../torch/csrc/jit/passes/onnx/constant_fold.cpp:179.) _C._jit_pass_onnx_graph_shape_type_inference( @@ -393,16 +417,18 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. quantized_out = torch.tensor(0.0, device=q_indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). for i, indices in enumerate(q_indices): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert (padding_left + padding_right) <= x.shape[-1] -Integrate OpenVINO to EnCodec pipeline --------------------------------------------------------------------------------- +Integrate OpenVINO to EnCodec pipeline +-------------------------------------- + + The following steps are required for integration of OpenVINO to EnCodec pipeline: @@ -412,8 +438,10 @@ pipeline: 3. Replace the original frame processing functions with OpenVINO based algorithms. -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -476,8 +504,10 @@ select device from dropdown list for running inference using OpenVINO model._encode_frame = encode_frame model._decode_frame = decode_frame -Run EnCodec with OpenVINO -------------------------------------------------------------------- +Run EnCodec with OpenVINO +------------------------- + + The process of running encodec with OpenVINO under hood will be the same like with the original PyTorch models. @@ -501,6 +531,13 @@ like with the original PyTorch models. out, out_sr = decompress(out_file.read_bytes()) + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") + + .. code:: ipython3 ov_output_file = "decopressed_ov.wav" @@ -521,7 +558,7 @@ like with the original PyTorch models. diff --git a/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_19_1.png b/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_19_1.png index e87ac388104..4d1f7be9418 100644 --- a/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_19_1.png +++ b/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_19_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:163c03d2e54146fc13d51ca270e2b8601a8545292cf8d2e62394f818ef754548 -size 44358 +oid sha256:c99a97c7779acef829103435a9de148f7d7ec1dea72b09b694e6c039f2737c45 +size 44357 diff --git a/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_38_1.png b/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_38_1.png index e87ac388104..4d1f7be9418 100644 --- a/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_38_1.png +++ b/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_38_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:163c03d2e54146fc13d51ca270e2b8601a8545292cf8d2e62394f818ef754548 -size 44358 +oid sha256:c99a97c7779acef829103435a9de148f7d7ec1dea72b09b694e6c039f2737c45 +size 44357 diff --git a/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_6_2.png b/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_6_2.png index 1c2d79f85f3..3a8316bb151 100644 --- a/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_6_2.png +++ b/docs/notebooks/234-encodec-audio-compression-with-output_files/234-encodec-audio-compression-with-output_6_2.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:008a0a166593dc371f3a6d5fb865a10fa23ddc589e342eaec7da6de1a1e0241d +oid sha256:f29c7e17e50adc3a28c221231ed536f5148c7d9c5534078d6bdef078632bcd07 size 45005 diff --git a/docs/notebooks/234-encodec-audio-compression-with-output_files/index.html b/docs/notebooks/234-encodec-audio-compression-with-output_files/index.html index e5b083f2014..c1a265b91e4 100644 --- a/docs/notebooks/234-encodec-audio-compression-with-output_files/index.html +++ b/docs/notebooks/234-encodec-audio-compression-with-output_files/index.html @@ -1,9 +1,9 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/234-encodec-audio-compression-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/234-encodec-audio-compression-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/234-encodec-audio-compression-with-output_files/


../
-234-encodec-audio-compression-with-output_19_1.png 31-Oct-2023 00:35               44358
-234-encodec-audio-compression-with-output_38_1.png 31-Oct-2023 00:35               44358
-234-encodec-audio-compression-with-output_6_2.png  31-Oct-2023 00:35               45005
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/234-encodec-audio-compression-with-output_files/


../
+234-encodec-audio-compression-with-output_19_1.png 07-Dec-2023 00:49               44357
+234-encodec-audio-compression-with-output_38_1.png 07-Dec-2023 00:49               44357
+234-encodec-audio-compression-with-output_6_2.png  07-Dec-2023 00:49               45005
 

diff --git a/docs/notebooks/235-controlnet-stable-diffusion-with-output.rst b/docs/notebooks/235-controlnet-stable-diffusion-with-output.rst index 55443ec56c1..14941bc5aa2 100644 --- a/docs/notebooks/235-controlnet-stable-diffusion-with-output.rst +++ b/docs/notebooks/235-controlnet-stable-diffusion-with-output.rst @@ -141,6 +141,7 @@ discussed steps are also applicable to other annotation modes. **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Instantiating Generation Pipeline <#instantiating-generation-pipeline>`__ @@ -175,7 +176,7 @@ Prerequisites .. code:: ipython3 %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "torch" "torchvision" - %pip install -q "diffusers>=0.14.0" "transformers>=4.30.2" "controlnet-aux>=0.0.6" "gradio>=3.36" + %pip install -q "diffusers>=0.14.0" "transformers>=4.30.2" "controlnet-aux>=0.0.6" "gradio>=3.36" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "openvino>=2023.1.0" Instantiating Generation Pipeline diff --git a/docs/notebooks/235-controlnet-stable-diffusion-with-output_files/index.html b/docs/notebooks/235-controlnet-stable-diffusion-with-output_files/index.html index 413af7ce9ee..ddfffdf4355 100644 --- a/docs/notebooks/235-controlnet-stable-diffusion-with-output_files/index.html +++ b/docs/notebooks/235-controlnet-stable-diffusion-with-output_files/index.html @@ -1,10 +1,10 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/235-controlnet-stable-diffusion-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/235-controlnet-stable-diffusion-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/235-controlnet-stable-diffusion-with-output_files/


../
-235-controlnet-stable-diffusion-with-output_17_..> 15-Nov-2023 00:43              498463
-235-controlnet-stable-diffusion-with-output_34_..> 15-Nov-2023 00:43               30487
-235-controlnet-stable-diffusion-with-output_34_..> 15-Nov-2023 00:43              464375
-235-controlnet-stable-diffusion-with-output_8_0..> 15-Nov-2023 00:43              498463
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/235-controlnet-stable-diffusion-with-output_files/


../
+235-controlnet-stable-diffusion-with-output_17_..> 07-Dec-2023 00:49              498463
+235-controlnet-stable-diffusion-with-output_34_..> 07-Dec-2023 00:49               30487
+235-controlnet-stable-diffusion-with-output_34_..> 07-Dec-2023 00:49              464375
+235-controlnet-stable-diffusion-with-output_8_0..> 07-Dec-2023 00:49              498463
 

diff --git a/docs/notebooks/236-stable-diffusion-v2-infinite-zoom-with-output.rst b/docs/notebooks/236-stable-diffusion-v2-infinite-zoom-with-output.rst index 30edc7d2c08..5e5411cc6f4 100644 --- a/docs/notebooks/236-stable-diffusion-v2-infinite-zoom-with-output.rst +++ b/docs/notebooks/236-stable-diffusion-v2-infinite-zoom-with-output.rst @@ -22,7 +22,7 @@ In previous notebooks, we already discussed how to run `Text-to-Image generation and Image-to-Image generation using Stable Diffusion v1 <225-stable-diffusion-text-to-image-with-output.html>`__ and `controlling its generation process using -ControlNet <235-controlnet-stable-diffusion/235-controlnet-stable-diffusion-with-output.html>`__. +ControlNet <./235-controlnet-stable-diffusion/235-controlnet-stable-diffusion.ipynb>`__. Now is turn of Stable Diffusion v2. Stable Diffusion v2: What’s new? @@ -86,17 +86,17 @@ Notebook contains the following steps: library <#stable-diffusion-in-diffusers-library>`__ - `Convert models to OpenVINO Intermediate representation (IR) format <#convert-models-to-openvino-intermediate-representation-ir-format>`__ - - `Prepare Inference - pipeline <#prepare-inference-pipeline>`__ + - `Prepare Inference pipeline <#prepare-inference-pipeline>`__ - `Zoom Video Generation <#zoom-video-generation>`__ - - `Configure Inference - Pipeline <#configure-inference-pipeline>`__ + - `Configure Inference Pipeline <#configure-inference-pipeline>`__ - `Select inference device <#select-inference-device>`__ - `Run Infinite Zoom video generation <#run-infinite-zoom-video-generation>`__ -Stable Diffusion v2 Infinite Zoom Showcase ------------------------------------------------------------------------------------- +Stable Diffusion v2 Infinite Zoom Showcase +------------------------------------------ + + In this tutorial we consider how to use Stable Diffusion v2 model for generation sequence of images for infinite zoom video effect. To do @@ -104,8 +104,10 @@ this, we will need `stabilityai/stable-diffusion-2-inpainting `__ model. -Stable Diffusion Text guided Inpainting -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Stable Diffusion Text guided Inpainting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + In image editing, inpainting is a process of restoring missing parts of pictures. Most commonly applied to reconstructing old deteriorated @@ -138,17 +140,21 @@ Using this inpainting feature, decreasing image by certain margin and masking this border for every new frame we can create interesting Zoom Out video based on our prompt. -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + install required packages .. code:: ipython3 - %pip install -q "diffusers>=0.14.0" "transformers >= 4.25.1" gradio "openvino>=2023.1.0" + %pip install -q "diffusers>=0.14.0" "transformers>=4.25.1" gradio "openvino>=2023.1.0" --extra-index-url https://download.pytorch.org/whl/cpu + +Stable Diffusion in Diffusers library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + -Stable Diffusion in Diffusers library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To work with Stable Diffusion v2, we will use Hugging Face `Diffusers `__ library. To @@ -199,11 +205,13 @@ The code below demonstrates how to create del pipe_inpaint gc.collect(); -Convert models to OpenVINO Intermediate representation (IR) format -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert models to OpenVINO Intermediate representation (IR) format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Conversion part of model stayed remain as in `Text-to-Image generation -notebook <236-stable-diffusion-v2-text-to-image-with-output.html>`__. Except +notebook <./236-stable-diffusion-v2-text-to-image.ipynb>`__. Except U-Net now has 9 channels, which now calculated like 4 for U-Net generated latents channels + 4 for latent representation of masked image + 1 channel resized mask. @@ -428,8 +436,10 @@ generated latents channels + 4 for latent representation of masked image VAE decoder will be loaded from sd2_inpainting/vae_decoder.xml -Prepare Inference pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare Inference pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + As it was discussed previously, Inpainting inference pipeline is based on Text-to-Image inference pipeline with addition mask processing step. @@ -445,7 +455,7 @@ We will reuse ``OVStableDiffusionPipeline`` basic utilities in import cv2 from transformers import CLIPTokenizer - from diffusers.pipeline_utils import DiffusionPipeline + from diffusers import DiffusionPipeline from diffusers.schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler @@ -500,8 +510,8 @@ We will reuse ``OVStableDiffusionPipeline`` basic utilities in .. parsed-literal:: - /tmp/ipykernel_1292073/2055396221.py:8: FutureWarning: Importing `DiffusionPipeline` or `ImagePipelineOutput` from diffusers.pipeline_utils is deprecated. Please import from diffusers.pipelines.pipeline_utils instead. - from diffusers.pipeline_utils import DiffusionPipeline + /tmp/ipykernel_1292073/2055396221.py:8: FutureWarning: Importing `DiffusionPipeline` or `ImagePipelineOutput` from diffusers is deprecated. Please import from diffusers.pipelines.pipeline_utils instead. + from diffusers import DiffusionPipeline .. code:: ipython3 @@ -860,8 +870,10 @@ We will reuse ``OVStableDiffusionPipeline`` basic utilities in return timesteps, num_inference_steps - t_start -Zoom Video Generation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Zoom Video Generation +~~~~~~~~~~~~~~~~~~~~~ + + For achieving zoom effect, we will use inpainting to expand images beyond their original borders. We run our @@ -1095,8 +1107,10 @@ generation is finished, we record frames in reversed order. loop=0, ) -Configure Inference Pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure Inference Pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Configuration steps: 1. Load models on device 2. Configure tokenizer and scheduler 3. Create instance of ``OVStableDiffusionInpaintingPipeline`` @@ -1108,8 +1122,10 @@ class tokenizer = CLIPTokenizer.from_pretrained('openai/clip-vit-large-patch14') -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -1154,8 +1170,10 @@ select device from dropdown list for running inference using OpenVINO scheduler=scheduler_inpaint, ) -Run Infinite Zoom video generation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Infinite Zoom video generation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -1218,3 +1236,5 @@ Run Infinite Zoom video generation .. .. raw:: html ..
+ + diff --git a/docs/notebooks/236-stable-diffusion-v2-text-to-image-demo-with-output.rst b/docs/notebooks/236-stable-diffusion-v2-text-to-image-demo-with-output.rst index 3c7ee85bcbb..1adc480041c 100644 --- a/docs/notebooks/236-stable-diffusion-v2-text-to-image-demo-with-output.rst +++ b/docs/notebooks/236-stable-diffusion-v2-text-to-image-demo-with-output.rst @@ -21,25 +21,21 @@ like to see the full implementation of stable diffusion for text to image, please visit `236-stable-diffusion-v2-text-to-image `__. - **Table of contents:** ---- - `Step 0: Install and import prerequisites <#step--install-and-import-prerequisites>`__ - `Step 1: Stable Diffusion v2 Fundamental components <#step--stable-diffusion-v-fundamental-components>`__ -- `Step 1.1: Retrieve components from HuggingFace <#step--retrieve-components-from-huggingface>`__ +- `Step 1.1: Retrieve components fromHuggingFace <#step--retrieve-components-from-huggingface>`__ - `Step 2: Convert the models to OpenVINO <#step--convert-the-models-to-openvino>`__ - `Step 3: Text-to-Image Generation Inference Pipeline <#step--text-to-image-generation-inference-pipeline>`__ - `Step 3.1: Load and Understand Text to Image OpenVINO models <#step--load-and-understand-text-to-image-openvino-models>`__ - `Step 3.2: Select inference device <#step--select-inference-device>`__ - `Step 3.3: Run Text-to-Image generation <#step--run-text-to-image-generation>`__ -Step 0: Install and import prerequisites ----------------------------------------------------------------------------------- +Step 0: Install and import prerequisites +---------------------------------------- -.. code:: ipython3 - from pathlib import Path To work with Stable Diffusion v2, we will use Hugging Face’s `Diffusers `__ library. @@ -51,7 +47,7 @@ pipelines `__. .. code:: ipython3 - %pip install -q "diffusers>=0.14.0" "openvino>=2023.1.0" "transformers >= 4.31" accelerate "urllib3==1.26.15" + %pip install -q "diffusers>=0.14.0" "openvino>=2023.1.0" "transformers>=4.31" accelerate "urllib3==1.26.15" --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -68,8 +64,10 @@ pipelines `__. Note: you may need to restart the kernel to use updated packages. -Step 1: Stable Diffusion v2 Fundamental components --------------------------------------------------------------------------------------------- +Step 1: Stable Diffusion v2 Fundamental components +-------------------------------------------------- + + Stable Diffusion pipelines for both Text to Image and Inpainting consist of three important parts: @@ -83,8 +81,10 @@ of three important parts: Depending on the pipeline, the parameters for these parts can differ, which we’ll explore in this demo! -Step 1.1: Retrieve components from HuggingFace -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Step 1.1: Retrieve components from HuggingFace +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Let’s start by retrieving these components from HuggingFace! @@ -125,8 +125,10 @@ using ``stable-diffusion-2-1``. Loading pipeline components...: 0%| | 0/6 [00:00 -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/236-stable-diffusion-v2-text-to-image-demo-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/236-stable-diffusion-v2-text-to-image-demo-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/236-stable-diffusion-v2-text-to-image-demo-with-output_files/


../
-236-stable-diffusion-v2-text-to-image-demo-with..> 31-Oct-2023 00:35              100414
-236-stable-diffusion-v2-text-to-image-demo-with..> 31-Oct-2023 00:35             1057851
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/236-stable-diffusion-v2-text-to-image-demo-with-output_files/


../
+236-stable-diffusion-v2-text-to-image-demo-with..> 07-Dec-2023 00:49              100414
+236-stable-diffusion-v2-text-to-image-demo-with..> 07-Dec-2023 00:49             1057851
 

diff --git a/docs/notebooks/236-stable-diffusion-v2-text-to-image-with-output.rst b/docs/notebooks/236-stable-diffusion-v2-text-to-image-with-output.rst index 882d1586f30..bb74f68a505 100644 --- a/docs/notebooks/236-stable-diffusion-v2-text-to-image-with-output.rst +++ b/docs/notebooks/236-stable-diffusion-v2-text-to-image-with-output.rst @@ -22,7 +22,7 @@ In previous notebooks, we already discussed how to run `Text-to-Image generation and Image-to-Image generation using Stable Diffusion v1 <225-stable-diffusion-text-to-image-with-output.html>`__ and `controlling its generation process using -ControlNet <235-controlnet-stable-diffusion-with-output.html>`__. +ControlNet <./235-controlnet-stable-diffusion/235-controlnet-stable-diffusion.ipynb>`__. Now is turn of Stable Diffusion v2. Stable Diffusion v2: What’s new? @@ -90,24 +90,25 @@ notebook `__ - `U-Net <#u-net>`__ - `VAE <#vae>`__ - - `Prepare Inference - Pipeline <#prepare-inference-pipeline>`__ - - `Configure Inference - Pipeline <#configure-inference-pipeline>`__ - - `Run Text-to-Image - generation <#run-text-to-image-generation>`__ + - `Prepare Inference Pipeline <#prepare-inference-pipeline>`__ + - `Configure Inference Pipeline <#configure-inference-pipeline>`__ + - `Run Text-to-Image generation <#run-text-to-image-generation>`__ + +Prerequisites +------------- + -Prerequisites -------------------------------------------------------- install required packages .. code:: ipython3 - %pip install -q "diffusers>=0.14.0" "openvino>=2023.1.0" "transformers >= 4.25.1" gradio + %pip install -q "diffusers>=0.14.0" "openvino>=2023.1.0" "transformers>=4.25.1" gradio --extra-index-url https://download.pytorch.org/whl/cpu + +Stable Diffusion v2 for Text-to-Image Generation +------------------------------------------------ + -Stable Diffusion v2 for Text-to-Image Generation ------------------------------------------------------------------------------------------- To start, let’s look on Text-to-Image process for Stable Diffusion v2. We will use `Stable Diffusion @@ -121,10 +122,11 @@ post `__ and original model `repository `__. -Stable Diffusion in Diffusers library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Stable Diffusion in Diffusers library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To work with Stable Diffusion v2, we will use Hugging Face +To work with Stable Diffusion +v2, we will use Hugging Face `Diffusers `__ library. To experiment with Stable Diffusion models, Diffusers exposes the `StableDiffusionPipeline `__ @@ -166,8 +168,10 @@ using ``stable-diffusion-2-1``: Fetching 13 files: 0%| | 0/13 [00:00`__ - `Image encoding <#image-encoding>`__ - `Example point input <#example-point-input>`__ - - `Example with multiple - points <#example-with-multiple-points>`__ + - `Example with multiple points <#example-with-multiple-points>`__ - `Example box and point input with negative label <#example-box-and-point-input-with-negative-label>`__ @@ -33,8 +32,7 @@ Object masks from prompts with SAM and OpenVINO - `Optimize encoder using NNCF Post-training Quantization API <#optimize-encoder-using-nncf-post-training-quantization-api>`__ - - `Prepare a calibration - dataset <#prepare-a-calibration-dataset>`__ + - `Prepare a calibration dataset <#prepare-a-calibration-dataset>`__ - `Run quantization and serialize OpenVINO IR model <#run-quantization-and-serialize-openvino-ir-model>`__ - `Validate Quantized Model @@ -66,8 +64,10 @@ zero-shot transfer). This notebook shows an example of how to convert and use Segment Anything Model in OpenVINO format, allowing it to run on a variety of platforms that support an OpenVINO. -Background ----------------------------------------------------- +Background +---------- + + Previously, to solve any kind of segmentation problem, there were two classes of approaches. The first, interactive segmentation, allowed for @@ -134,18 +134,24 @@ post =3.25" "openvino>=2023.1.0" "nncf>=2.5.0" + %pip install -q "segment_anything" "gradio>=3.25" "openvino>=2023.1.0" "nncf>=2.5.0" "torch>=2.1" "torchvision>=0.16" --extra-index-url https://download.pytorch.org/whl/cpu + +Convert model to OpenVINO Intermediate Representation +----------------------------------------------------- + + + +Download model checkpoint and create PyTorch model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Convert model to OpenVINO Intermediate Representation ------------------------------------------------------------------------------------------------ -Download model checkpoint and create PyTorch model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There are several Segment Anything Model `checkpoints `__ @@ -195,8 +201,10 @@ into account this fact, we split model on 2 independent parts: image_encoder and mask_predictor (combination of Prompt Encoder and Mask Decoder). -Image Encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image Encoder +~~~~~~~~~~~~~ + + Image Encoder input is tensor with shape ``1x3x1024x1024`` in ``NCHW`` format, contains image for segmentation. Image Encoder output is image @@ -248,8 +256,10 @@ embeddings, tensor with shape ``1x256x64x64`` ov_encoder = core.compile_model(ov_encoder_model, device.value) -Mask predictor -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Mask predictor +~~~~~~~~~~~~~~ + + This notebook expects the model was exported with the parameter ``return_single_mask=True``. It means that model will only return the @@ -425,11 +435,15 @@ Model outputs: ov_predictor = core.compile_model(ov_model, device.value) -Run OpenVINO model in interactive segmentation mode ---------------------------------------------------------------------------------------------- +Run OpenVINO model in interactive segmentation mode +--------------------------------------------------- + + + +Example Image +~~~~~~~~~~~~~ + -Example Image -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: ipython3 @@ -459,8 +473,10 @@ Example Image .. image:: 237-segment-anything-with-output_files/237-segment-anything-with-output_21_0.png -Preprocessing and visualization utilities -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Preprocessing and visualization utilities +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + To prepare input for Image Encoder we should: @@ -575,8 +591,10 @@ These steps are applicable to all available models w, h = box[2] - box[0], box[3] - box[1] ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0, 0, 0, 0), lw=2)) -Image encoding -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image encoding +~~~~~~~~~~~~~~ + + To start work with image, we should preprocess it and obtain image embeddings using ``ov_encoder``. We will use the same image for all @@ -592,8 +610,10 @@ reuse them. Now, we can try to provide different prompts for mask generation -Example point input -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Example point input +~~~~~~~~~~~~~~~~~~~ + + In this example we select one point. The green star symbol show its location on the image below. @@ -658,8 +678,10 @@ object). .. image:: 237-segment-anything-with-output_files/237-segment-anything-with-output_35_0.png -Example with multiple points -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Example with multiple points +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + in this example, we provide additional point for cover larger object area. @@ -725,8 +747,10 @@ Package inputs, then predict and threshold the mask. Great! Looks like now, predicted mask cover whole truck. -Example box and point input with negative label -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Example box and point input with negative label +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + In this example we define input prompt using bounding box and point inside it.The bounding box represented as set of points of its left @@ -798,8 +822,10 @@ Package inputs, then predict and threshold the mask. .. image:: 237-segment-anything-with-output_files/237-segment-anything-with-output_53_0.png -Interactive segmentation ------------------------------------------------------------------- +Interactive segmentation +------------------------ + + Now, you can try SAM on own image. Upload image to input window and click on desired point, model predict segment based on your image and @@ -852,8 +878,8 @@ point. with gr.Blocks() as demo: with gr.Row(): - input_img = gr.Image(label="Input", type="numpy").style(height=480, width=480) - output_img = gr.Image(label="Selected Segment", type="numpy").style(height=480, width=480) + input_img = gr.Image(label="Input", type="numpy", height=480, width=480) + output_img = gr.Image(label="Selected Segment", type="numpy", height=480, width=480) def on_image_change(img): segmenter.set_image(img) @@ -885,14 +911,6 @@ point. demo.launch(share=True) -.. parsed-literal:: - - /tmp/ipykernel_862585/1907223323.py:46: GradioDeprecationWarning: The `style` method is deprecated. Please set these arguments in the constructor instead. - input_img = gr.Image(label="Input", type="numpy").style(height=480, width=480) - /tmp/ipykernel_862585/1907223323.py:47: GradioDeprecationWarning: The `style` method is deprecated. Please set these arguments in the constructor instead. - output_img = gr.Image(label="Selected Segment", type="numpy").style(height=480, width=480) - - .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 @@ -906,8 +924,10 @@ point. ..
-Run OpenVINO model in automatic mask generation mode ----------------------------------------------------------------------------------------------- +Run OpenVINO model in automatic mask generation mode +---------------------------------------------------- + + Since SAM can efficiently process prompts, masks for the entire image can be generated by sampling a large number of prompts over an image. @@ -1273,8 +1293,10 @@ is a dictionary containing various data about the mask. These keys are: -Optimize encoder using NNCF Post-training Quantization API ----------------------------------------------------------------------------------------------------- +Optimize encoder using NNCF Post-training Quantization API +---------------------------------------------------------- + + `NNCF `__ provides a suite of advanced algorithms for Neural Networks inference optimization in @@ -1291,8 +1313,10 @@ The optimization process contains the following steps: 3. Serialize OpenVINO IR model, using the ``openvino.save_model`` function. -Prepare a calibration dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare a calibration dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Download COCO dataset. Since the dataset is used to calibrate the model’s parameter instead of fine-tuning it, we don’t need to download @@ -1368,8 +1392,10 @@ dataset and returns data that can be passed to the model for inference. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -Run quantization and serialize OpenVINO IR model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run quantization and serialize OpenVINO IR model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The ``nncf.quantize`` function provides an interface for model quantization. It requires an instance of the OpenVINO Model and @@ -1431,8 +1457,10 @@ activations. ov_encoder_path_int8 = "sam_image_encoder_int8.xml" ov.save_model(quantized_model, ov_encoder_path_int8) -Validate Quantized Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate Quantized Model Inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + We can reuse the previous code to validate the output of ``INT8`` model. @@ -1495,10 +1523,11 @@ Run ``INT8`` model in automatic mask generation mode -Compare Performance of the Original and Quantized Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Compare Performance of the Original and Quantized Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Finally, use the OpenVINO `Benchmark +Finally, use the OpenVINO +`Benchmark Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. diff --git a/docs/notebooks/237-segment-anything-with-output_files/index.html b/docs/notebooks/237-segment-anything-with-output_files/index.html index 71c3c02fad6..6d8401dae2a 100644 --- a/docs/notebooks/237-segment-anything-with-output_files/index.html +++ b/docs/notebooks/237-segment-anything-with-output_files/index.html @@ -1,18 +1,18 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/237-segment-anything-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/237-segment-anything-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/237-segment-anything-with-output_files/


../
-237-segment-anything-with-output_21_0.png          31-Oct-2023 00:35              467418
-237-segment-anything-with-output_28_0.png          31-Oct-2023 00:35              468529
-237-segment-anything-with-output_35_0.png          31-Oct-2023 00:35              469443
-237-segment-anything-with-output_39_0.png          31-Oct-2023 00:35              470668
-237-segment-anything-with-output_44_0.png          31-Oct-2023 00:35              468092
-237-segment-anything-with-output_48_0.png          31-Oct-2023 00:35              468088
-237-segment-anything-with-output_53_0.png          31-Oct-2023 00:35              472756
-237-segment-anything-with-output_68_1.jpg          31-Oct-2023 00:35              262203
-237-segment-anything-with-output_68_1.png          31-Oct-2023 00:35             2409333
-237-segment-anything-with-output_80_0.png          31-Oct-2023 00:35              469432
-237-segment-anything-with-output_82_1.jpg          31-Oct-2023 00:35              262535
-237-segment-anything-with-output_82_1.png          31-Oct-2023 00:35             2397126
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/237-segment-anything-with-output_files/


../
+237-segment-anything-with-output_21_0.png          07-Dec-2023 00:49              467418
+237-segment-anything-with-output_28_0.png          07-Dec-2023 00:49              468529
+237-segment-anything-with-output_35_0.png          07-Dec-2023 00:49              469443
+237-segment-anything-with-output_39_0.png          07-Dec-2023 00:49              470668
+237-segment-anything-with-output_44_0.png          07-Dec-2023 00:49              468092
+237-segment-anything-with-output_48_0.png          07-Dec-2023 00:49              468088
+237-segment-anything-with-output_53_0.png          07-Dec-2023 00:49              472756
+237-segment-anything-with-output_68_1.jpg          07-Dec-2023 00:49              262203
+237-segment-anything-with-output_68_1.png          07-Dec-2023 00:49             2409333
+237-segment-anything-with-output_80_0.png          07-Dec-2023 00:49              469432
+237-segment-anything-with-output_82_1.jpg          07-Dec-2023 00:49              262535
+237-segment-anything-with-output_82_1.png          07-Dec-2023 00:49             2397126
 

diff --git a/docs/notebooks/239-image-bind-convert-with-output.rst b/docs/notebooks/239-image-bind-convert-with-output.rst index 0fee6dbdeb7..2d3ea2a420a 100644 --- a/docs/notebooks/239-image-bind-convert-with-output.rst +++ b/docs/notebooks/239-image-bind-convert-with-output.rst @@ -83,19 +83,20 @@ zero-shot classification. - `Text-Image classification <#text-image-classification>`__ - `Text-Audio classification <#text-audio-classification>`__ - - `Image-Audio - classification <#image-audio-classification>`__ + - `Image-Audio classification <#image-audio-classification>`__ - `Next Steps <#next-steps>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + .. code:: ipython3 import sys - %pip install -q soundfile pytorchvideo ftfy "timm==0.6.7" einops fvcore "openvino>=2023.1.0" + %pip install -q soundfile pytorchvideo ftfy "timm==0.6.7" einops fvcore "openvino>=2023.1.0" --extra-index-url https://download.pytorch.org/whl/cpu if sys.version_info.minor < 8: %pip install -q "decord" @@ -124,8 +125,10 @@ Prerequisites /home/ea/work/openvino_notebooks/notebooks/239-image-bind/ImageBind -Instantiate PyTorch model -------------------------------------------------------------------- +Instantiate PyTorch model +------------------------- + + To start work with the model, we should instantiate the PyTorch model class. ``imagebind_model.imagebind_huge(pretrained=True)`` downloads @@ -160,8 +163,10 @@ card `__. warnings.warn( -Prepare input data ------------------------------------------------------------- +Prepare input data +------------------ + + ImageBind works with data across 6 different modalities. Each of them requires its steps for preprocessing. ``data`` module is responsible for @@ -192,8 +197,10 @@ data reading and preprocessing for each modality. ModalityType.AUDIO: data.load_and_transform_audio_data(audio_paths, "cpu"), } -Convert Model to OpenVINO Intermediate Representation (IR) format ------------------------------------------------------------------------------------------------------------ +Convert Model to OpenVINO Intermediate Representation (IR) format +----------------------------------------------------------------- + + OpenVINO supports PyTorch through Model Conversion API. You will use `model conversion Python @@ -226,8 +233,10 @@ embeddings. core = ov.Core() -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -289,8 +298,10 @@ select device from dropdown list for running inference using OpenVINO if npatch_per_img == N: -Zero-shot classification using ImageBind and OpenVINO ------------------------------------------------------------------------------------------------ +Zero-shot classification using ImageBind and OpenVINO +----------------------------------------------------- + + In zero-shot classification, a piece of data is embedded and fed to the model to retrieve a label that corresponds with the contents of the @@ -350,8 +361,10 @@ they represent the same object. image_list = [img.split('/')[-1] for img in image_paths] audio_list = [audio.split('/')[-1] for audio in audio_paths] -Text-Image classification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text-Image classification +~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -364,8 +377,10 @@ Text-Image classification .. image:: 239-image-bind-convert-with-output_files/239-image-bind-convert-with-output_20_0.png -Text-Audio classification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text-Audio classification +~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -378,8 +393,10 @@ Text-Audio classification .. image:: 239-image-bind-convert-with-output_files/239-image-bind-convert-with-output_22_0.png -Image-Audio classification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image-Audio classification +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -491,9 +508,11 @@ Putting all together, we can match text, image, and sound for our data. -Next Steps ----------------------------------------------------- +Next Steps +---------- -Open the `239-image-bind-quantize <239-image-bind-quantize-with-output.html>`__ + + +Open the `239-image-bind-quantize <239-image-bind-quantize.ipynb>`__ notebook to quantize the IR model with the Post-training Quantization API of NNCF and compare ``FP16`` and ``INT8`` models. diff --git a/docs/notebooks/239-image-bind-convert-with-output_files/index.html b/docs/notebooks/239-image-bind-convert-with-output_files/index.html index 3e1c221be0c..748ce6aba03 100644 --- a/docs/notebooks/239-image-bind-convert-with-output_files/index.html +++ b/docs/notebooks/239-image-bind-convert-with-output_files/index.html @@ -1,15 +1,15 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/239-image-bind-convert-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/239-image-bind-convert-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/239-image-bind-convert-with-output_files/


../
-239-image-bind-convert-with-output_20_0.png        31-Oct-2023 00:35               15474
-239-image-bind-convert-with-output_22_0.png        31-Oct-2023 00:35               13795
-239-image-bind-convert-with-output_24_0.png        31-Oct-2023 00:35               18151
-239-image-bind-convert-with-output_26_1.jpg        31-Oct-2023 00:35               36700
-239-image-bind-convert-with-output_26_1.png        31-Oct-2023 00:35              341289
-239-image-bind-convert-with-output_27_1.jpg        31-Oct-2023 00:35               71448
-239-image-bind-convert-with-output_27_1.png        31-Oct-2023 00:35              839471
-239-image-bind-convert-with-output_28_1.jpg        31-Oct-2023 00:35               54208
-239-image-bind-convert-with-output_28_1.png        31-Oct-2023 00:35              658748
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/239-image-bind-convert-with-output_files/


../
+239-image-bind-convert-with-output_20_0.png        07-Dec-2023 00:49               15474
+239-image-bind-convert-with-output_22_0.png        07-Dec-2023 00:49               13795
+239-image-bind-convert-with-output_24_0.png        07-Dec-2023 00:49               18151
+239-image-bind-convert-with-output_26_1.jpg        07-Dec-2023 00:49               36700
+239-image-bind-convert-with-output_26_1.png        07-Dec-2023 00:49              341289
+239-image-bind-convert-with-output_27_1.jpg        07-Dec-2023 00:49               71448
+239-image-bind-convert-with-output_27_1.png        07-Dec-2023 00:49              839471
+239-image-bind-convert-with-output_28_1.jpg        07-Dec-2023 00:49               54208
+239-image-bind-convert-with-output_28_1.png        07-Dec-2023 00:49              658748
 

diff --git a/docs/notebooks/240-dolly-2-instruction-following-with-output.rst b/docs/notebooks/240-dolly-2-instruction-following-with-output.rst index 3dca6e68489..552b8a95b99 100644 --- a/docs/notebooks/240-dolly-2-instruction-following-with-output.rst +++ b/docs/notebooks/240-dolly-2-instruction-following-with-output.rst @@ -83,6 +83,7 @@ and `repo `__ **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Select inference device <#select-inference-device>`__ @@ -120,7 +121,7 @@ documentation `__. .. code:: ipython3 - %pip install -q "diffusers>=0.16.1" "transformers>=4.33.0" "openvino==2023.2.0.dev20230922" "nncf>=2.6.0" datasets onnx gradio --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "diffusers>=0.16.1" "transformers>=4.33.0" "openvino>=2023.2.0" "nncf>=2.6.0" datasets onnx gradio --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q --upgrade "git+https://github.com/huggingface/optimum-intel.git" Select inference device @@ -218,10 +219,10 @@ compatible with Optimum models. .. parsed-literal:: No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - 2023-10-09 11:07:22.234444: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-09 11:07:22.273745: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-11-17 13:10:43.359093: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-11-17 13:10:43.398436: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-09 11:07:22.903943: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-11-17 13:10:44.026743: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Compiling the model to CPU ... @@ -282,7 +283,6 @@ accuracy drop. if to_compress.value: if not compressed_model_path.exists(): - ov_model = OVModelForCausalLM.from_pretrained(model_id, device=current_device, export=True, ov_config=ov_config) quantizer = OVQuantizer.from_pretrained(ov_model) quantizer.quantize(save_directory=compressed_model_path, weights_only=True) del quantizer @@ -295,8 +295,8 @@ accuracy drop. .. parsed-literal:: * Original IR model size: 5297.21 MB - * Compressed IR model size: 2660.29 MB - * Model compression rate: 1.991 + * Compressed IR model size: 2657.89 MB + * Model compression rate: 1.993 .. parsed-literal:: @@ -717,23 +717,3 @@ generation parameters: # If you are launching remotely, specify server_name and server_port # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` # To learn more please refer to the Gradio docs: https://gradio.app/docs/ - - -.. parsed-literal:: - - /tmp/ipykernel_709262/2332051390.py:57: GradioDeprecationWarning: The `enable_queue` parameter has been deprecated. Please use the `.queue()` method instead. - demo.launch(enable_queue=True, share=False, height=800) - - -.. parsed-literal:: - - Running on local URL: http://127.0.0.1:7860 - - To create a public link, set `share=True` in `launch()`. - - - -.. .. raw:: html - -..
- diff --git a/docs/notebooks/245-typo-detector-with-output.rst b/docs/notebooks/245-typo-detector-with-output.rst index 42ebf793637..ede76147a68 100644 --- a/docs/notebooks/245-typo-detector-with-output.rst +++ b/docs/notebooks/245-typo-detector-with-output.rst @@ -43,43 +43,41 @@ The model has been pretrained on the - `2. Converting the model to OpenVINO IR <#-converting-the-model-to-openvino-ir>`__ -- `Select inference device <#select-inference-device>`__ -- `1. Hugging Face Optimum Intel - library <#-hugging-face-optimum-intel-library>`__ + - `Select inference device <#select-inference-device>`__ - - `Load the model <#load-the-model>`__ - - `Load the tokenizer <#load-the-tokenizer>`__ + - `1. Hugging Face Optimum Intel + library <#-hugging-face-optimum-intel-library>`__ -- `2. Converting the model to OpenVINO - IR <#-converting-the-model-to-openvino-ir>`__ + - `Load the model <#load-the-model>`__ + - `Load the tokenizer <#load-the-tokenizer>`__ - - `Load the Pytorch model <#load-the-pytorch-model>`__ - - `Converting to OpenVINO IR <#converting-to-openvino-ir>`__ - - `Inference <#inference>`__ + - `2. Converting the model to OpenVINO + IR <#-converting-the-model-to-openvino-ir>`__ -- `Helper Functions <#helper-functions>`__ + - `Load the Pytorch model <#load-the-pytorch-model>`__ + - `Converting to OpenVINO IR <#converting-to-openvino-ir>`__ + - `Inference <#inference>`__ + + - `Helper Functions <#helper-functions>`__ .. code:: ipython3 - %pip install -q "diffusers>=0.17.1" "openvino>=2023.1.0" "nncf>=2.5.0" "gradio" "onnx>=1.11.0" "onnxruntime>=1.14.0" "transformers>=4.31.0" + %pip install -q "diffusers>=0.17.1" "openvino>=2023.1.0" "nncf>=2.5.0" "gradio" "onnx>=1.11.0" "transformers>=4.33.0" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/huggingface/optimum-intel.git" .. parsed-literal:: DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - onnxconverter-common 1.14.0 requires protobuf==3.20.2, but you have protobuf 4.24.4 which is incompatible. - pytorch-lightning 1.6.5 requires protobuf<=3.20.1, but you have protobuf 4.24.4 which is incompatible. - tensorflow 2.13.1 requires typing-extensions<4.6.0,>=3.6.6, but you have typing-extensions 4.8.0 which is incompatible. - tf2onnx 1.15.1 requires protobuf~=3.20.2, but you have protobuf 4.24.4 which is incompatible. Note: you may need to restart the kernel to use updated packages. DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. -Imports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Imports +~~~~~~~ + + .. code:: ipython3 @@ -93,14 +91,16 @@ Imports .. parsed-literal:: - 2023-10-31 00:01:48.550893: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-31 00:01:48.584584: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-07 00:04:25.014506: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-07 00:04:25.048142: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-31 00:01:49.140201: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-07 00:04:25.652741: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + +Methods +~~~~~~~ -Methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The notebook provides two methods to run the inference of typo detector with OpenVINO runtime, so that you can experience both calling the API @@ -108,8 +108,10 @@ of Optimum with OpenVINO Runtime included, and loading models in other frameworks, converting them to OpenVINO IR format, and running inference with OpenVINO Runtime. -1. Using the `Hugging Face Optimum `__ library -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +1. Using the `Hugging Face Optimum `__ library +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + + The Hugging Face Optimum API is a high-level API that allows us to convert models from the Hugging Face Transformers library to the @@ -117,8 +119,10 @@ OpenVINO™ IR format. Compiled models in OpenVINO IR format can be loaded using Optimum. Optimum allows the use of optimization on targeted hardware. -2. Converting the model to OpenVINO IR -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +2. Converting the model to OpenVINO IR +'''''''''''''''''''''''''''''''''''''' + + The Pytorch model is converted to `OpenVINO IR format `__. This @@ -147,8 +151,10 @@ methods | inference with OpenVINO Runtime | | +-----------------------------------+----------------------------------+ -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -177,8 +183,10 @@ select device from dropdown list for running inference using OpenVINO -1. Hugging Face Optimum Intel library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1. Hugging Face Optimum Intel library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + For this method, we need to install the ``Hugging Face Optimum Intel library`` accelerated by OpenVINO @@ -207,12 +215,12 @@ Import required model class .. parsed-literal:: No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations - warnings.warn( -Load the model -'''''''''''''''''''''''''''''''''''''''''''''''''''''''' +Load the model +'''''''''''''' + + From the ``OVModelForTokenCLassification`` class we will import the relevant pre-trained model. To load a Transformers model and convert it @@ -238,8 +246,8 @@ your model. Framework not specified. Using pt to export to ONNX. Using the export variant default. Available variants are: - - default: The default ONNX variant. - Using framework PyTorch: 1.13.1+cpu + - default: The default ONNX variant. + Using framework PyTorch: 2.1.1+cpu .. parsed-literal:: @@ -250,14 +258,15 @@ your model. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:74: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:75: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. op1 = operator(\*args, \*\*kwargs) Compiling the model to AUTO ... - Set CACHE_DIR to /tmp/tmpuz_oy32n/model_cache -Load the tokenizer -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +Load the tokenizer +'''''''''''''''''' + + Text Preprocessing cleans the text-based input data so it can be fed into the model. Tokenization splits paragraphs and sentences into @@ -362,14 +371,18 @@ Let’s run a demo using the Hugging Face Optimum API. [Input]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. [Detected]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. ---------------------------------------------------------------------------------------------------------------------------------- - Time elapsed: 0.20258617401123047 + Time elapsed: 0.1588735580444336 -2. Converting the model to OpenVINO IR -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +2. Converting the model to OpenVINO IR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +Load the Pytorch model +'''''''''''''''''''''' + -Load the Pytorch model -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' Use the ``AutoModelForTokenClassification`` class to load the pretrained pytorch model. @@ -389,8 +402,10 @@ pytorch model. model = AutoModelForTokenClassification.from_pretrained(model_id, config=config) model.save_pretrained(model_dir) -Converting to OpenVINO IR -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +Converting to OpenVINO IR +''''''''''''''''''''''''' + + .. code:: ipython3 @@ -400,8 +415,10 @@ Converting to OpenVINO IR ov_model = ov.convert_model(model, example_input=dict(dummy_model_input)) ov.save_model(ov_model, ov_model_path) -Inference -''''''''''''''''''''''''''''''''''''''''''''''''''' +Inference +''''''''' + + OpenVINO™ Runtime Python API is used to compile the model in OpenVINO IR format. The Core class from the ``openvino`` module is imported first. @@ -415,8 +432,10 @@ the compiled model as it is needed for inference. compiled_model = core.compile_model(ov_model, device.value) output_layer = compiled_model.output(0) -Helper Functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Helper Functions +~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -590,5 +609,5 @@ Let’s run a demo using the converted OpenVINO IR model. [Input]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. [Detected]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. ---------------------------------------------------------------------------------------------------------------------------------- - Time elapsed: 0.10554790496826172 + Time elapsed: 0.10169100761413574 diff --git a/docs/notebooks/247-code-language-id-with-output.rst b/docs/notebooks/247-code-language-id-with-output.rst index 5b982bd26ce..50d7cec212b 100644 --- a/docs/notebooks/247-code-language-id-with-output.rst +++ b/docs/notebooks/247-code-language-id-with-output.rst @@ -16,6 +16,7 @@ navigation. **Table of contents:** + - `Introduction <#introduction>`__ - `Task <#task>`__ @@ -116,14 +117,14 @@ Install prerequisites -First, complete the `repository installation steps <../notebooks_installation.html>`__. +First, complete the `repository installation steps <../../README.md>`__. Then, the following cell will install: - HuggingFace Optimum with OpenVINO support - HuggingFace Evaluate to benchmark results .. code:: ipython3 - %pip install -q "diffusers>=0.17.1" "openvino>=2023.1.0" "nncf>=2.5.0" "gradio" "onnx>=1.11.0" "transformers>=4.33.0" "evaluate" + %pip install -q "diffusers>=0.17.1" "openvino>=2023.1.0" "nncf>=2.5.0" "gradio" "onnx>=1.11.0" "transformers>=4.33.0" "evaluate" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/huggingface/optimum-intel.git" @@ -131,9 +132,9 @@ OpenVINO support - HuggingFace Evaluate to benchmark results DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - onnxconverter-common 1.14.0 requires protobuf==3.20.2, but you have protobuf 4.25.0 which is incompatible. - pytorch-lightning 1.6.5 requires protobuf<=3.20.1, but you have protobuf 4.25.0 which is incompatible. - tf2onnx 1.15.1 requires protobuf~=3.20.2, but you have protobuf 4.25.0 which is incompatible. + onnxconverter-common 1.14.0 requires protobuf==3.20.2, but you have protobuf 4.25.1 which is incompatible. + pytorch-lightning 1.6.5 requires protobuf<=3.20.1, but you have protobuf 4.25.1 which is incompatible. + tf2onnx 1.15.1 requires protobuf~=3.20.2, but you have protobuf 4.25.1 which is incompatible. Note: you may need to restart the kernel to use updated packages. DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. @@ -163,10 +164,10 @@ equivalent to ``AutoModelForSequenceClassification`` from Transformers .. parsed-literal:: - 2023-11-15 00:06:22.342451: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-11-15 00:06:22.376717: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-07 00:07:02.218482: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-07 00:07:02.252471: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-11-15 00:06:22.962059: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-07 00:07:02.836089: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -260,7 +261,7 @@ Download resources .. parsed-literal:: Framework not specified. Using pt to export to ONNX. - Some weights of the model checkpoint at huggingface/CodeBERTa-language-id were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias'] + Some weights of the model checkpoint at huggingface/CodeBERTa-language-id were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] - This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). Using the export variant default. Available variants are: @@ -283,7 +284,7 @@ Download resources .. parsed-literal:: - Ressources cached locally at: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/notebooks/247-code-language-id/model/CodeBERTa-language-id + Ressources cached locally at: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/247-code-language-id/model/CodeBERTa-language-id Create inference pipeline @@ -396,7 +397,7 @@ NOTE: the base model is loaded using .. parsed-literal:: - Some weights of the model checkpoint at huggingface/CodeBERTa-language-id were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias'] + Some weights of the model checkpoint at huggingface/CodeBERTa-language-id were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] - This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). @@ -582,7 +583,7 @@ Inference on new input using quantized model df['speed'] = df.distance / df.time Predicted label: python - Predicted score: 0.83 + Predicted score: 0.81 Load evaluation set @@ -699,16 +700,16 @@ displayed. base 1.0 - 2.340578 - 51.269396 - 0.019505 + 2.045702 + 58.659569 + 0.017048 quantized 1.0 - 3.334829 - 35.983857 - 0.027790 + 2.602893 + 46.102553 + 0.021691 @@ -719,8 +720,11 @@ displayed. Additional resources -------------------- - - `Grammatical Error Correction with OpenVINO `__ -- `Quantize a Hugging Face Question-Answering Model with OpenVINO `__ \ \*\* +- `Grammatical Error Correction +with +OpenVINO `__ +- `Quantize a Hugging Face Question-Answering Model with +OpenVINO `__\ \*\* Clean up -------- diff --git a/docs/notebooks/248-ssd-b1-with-output.rst b/docs/notebooks/248-ssd-b1-with-output.rst new file mode 100644 index 00000000000..0827e1ed950 --- /dev/null +++ b/docs/notebooks/248-ssd-b1-with-output.rst @@ -0,0 +1,336 @@ +Image generation with Segmind Stable Diffusion 1B (SSD-1B) model and OpenVINO +============================================================================= + +The `Segmind Stable Diffusion Model +(SSD-1B) `__ is +a distilled 50% smaller version of the Stable Diffusion XL (SDXL), +offering a 60% speedup while maintaining high-quality text-to-image +generation capabilities. It has been trained on diverse datasets, +including Grit and Midjourney scrape data, to enhance its ability to +create a wide range of visual content based on textual prompts. This +model employs a knowledge distillation strategy, where it leverages the +teachings of several expert models in succession, including SDXL, +ZavyChromaXL, and JuggernautXL, to combine their strengths and produce +impressive visual outputs. + +.. figure:: https://user-images.githubusercontent.com/82945616/277419571-a5583e8a-6a05-4680-a540-f80502feed0b.png + :alt: image + + image + +In this tutorial, we consider how to run the SSD-1B model using +OpenVINO. + +We will use a pre-trained model from the `Hugging Face +Diffusers `__ library. To +simplify the user experience, the `Hugging Face Optimum +Intel `__ library is +used to convert the models to OpenVINO™ IR format. + +**Table of contents:** + + +- `Install Prerequisites <#install-prerequisites>`__ +- `SSD-1B Base model <#ssd-b-base-model>`__ +- `Select inference device SSD-1B Base + model <#select-inference-device-ssd-b-base-model>`__ +- `Text2image Generation Interactive + Demo <#textimage-generation-interactive-demo>`__ + +Install prerequisites +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: ipython3 + + %pip install -q "git+https://github.com/huggingface/optimum-intel.git" + %pip install -q "openvino>=2023.1.0" + %pip install -q --upgrade-strategy eager "invisible-watermark>=0.2.0" "transformers>=4.33" "accelerate" "onnx" "onnxruntime" safetensors "diffusers>=0.22.0" + %pip install -q gradio + + +.. parsed-literal:: + + DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + Note: you may need to restart the kernel to use updated packages. + DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + Note: you may need to restart the kernel to use updated packages. + DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + Note: you may need to restart the kernel to use updated packages. + DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + Note: you may need to restart the kernel to use updated packages. + + +SSD-1B Base model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We will start with the base model part, which is responsible for the +generation of images of the desired output size. +`SSD-1B `__ is available for +downloading via the `HuggingFace hub `__. +It already provides a ready-to-use model in OpenVINO format compatible +with `Optimum +Intel `__. + +To load an OpenVINO model and run an inference with OpenVINO Runtime, +you need to replace diffusers ``StableDiffusionXLPipeline`` with Optimum +``OVStableDiffusionXLPipeline``. In case you want to load a PyTorch +model and convert it to the OpenVINO format on the fly, you can set +``export=True``. + +You can save the model on disk using the ``save_pretrained`` method. + +.. code:: ipython3 + + from pathlib import Path + from optimum.intel.openvino import OVStableDiffusionXLPipeline + + + model_id = "segmind/SSD-1B" + model_dir = Path("openvino-ssd-1b") + + +.. parsed-literal:: + + INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + + +.. parsed-literal:: + + No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' + 2023-12-07 00:09:54.638748: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-07 00:09:54.672777: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2023-12-07 00:09:55.202678: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + +Select inference device SSD-1B Base model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +select device from dropdown list for running inference using OpenVINO + +.. code:: ipython3 + + import ipywidgets as widgets + import openvino as ov + + + core = ov.Core() + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +.. code:: ipython3 + + import gc + + + if not model_dir.exists(): + text2image_pipe = OVStableDiffusionXLPipeline.from_pretrained(model_id, compile=False, device=device.value, export=True) + text2image_pipe.half() + text2image_pipe.save_pretrained(model_dir) + text2image_pipe.compile() + gc.collect() + else: + text2image_pipe = OVStableDiffusionXLPipeline.from_pretrained(model_dir, device=device.value) + + +.. parsed-literal:: + + Framework not specified. Using pt to export to ONNX. + Keyword arguments {'subfolder': '', 'trust_remote_code': False} are not expected by StableDiffusionXLImg2ImgPipeline and will be ignored. + + + +.. parsed-literal:: + + Loading pipeline components...: 0%| | 0/7 [00:00 1 or self.sliding_window is not None: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:137: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if past_key_values_length > 0: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:273: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:281: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:313: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + Using framework PyTorch: 1.13.1+cpu + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unet_2d_condition.py:878: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if dim % default_overall_up_factor != 0: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/resnet.py:265: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + assert hidden_states.shape[1] == self.channels + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/resnet.py:271: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + assert hidden_states.shape[1] == self.channels + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/resnet.py:173: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + assert hidden_states.shape[1] == self.channels + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/resnet.py:186: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if hidden_states.shape[0] >= 64: + Using framework PyTorch: 1.13.1+cpu + Using framework PyTorch: 1.13.1+cpu + Using framework PyTorch: 1.13.1+cpu + Compiling the vae_decoder to AUTO ... + Compiling the unet to AUTO ... + Compiling the vae_encoder to AUTO ... + Compiling the text_encoder_2 to AUTO ... + Compiling the text_encoder to AUTO ... + + +Run Text2Image generation pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now, we can run the model for the generation of images using text +prompts. To speed up evaluation and reduce the required memory we +decrease ``num_inference_steps`` and image size (using ``height`` and +``width``). You can modify them to suit your needs and depend on the +target hardware. We also specified a ``generator`` parameter based on a +numpy random state with a specific seed for results reproducibility. +>\ **Note**: Generating a default size 1024x1024 image requires about +53GB for the SSD-1B model in case if the converted model is loaded from +disk and up to 64GB RAM for the SDXL model after exporting. + +.. code:: ipython3 + + prompt = "An astronaut riding a green horse" # Your prompt here + neg_prompt = "ugly, blurry, poor quality" # Negative prompt here + image = text2image_pipe(prompt=prompt, num_inference_steps=15, negative_prompt=neg_prompt).images[0] + image + + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/intel/openvino/modeling_diffusion.py:565: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + outputs = self.request(inputs, shared_memory=True) + + + +.. parsed-literal:: + + 0%| | 0/15 [00:00 + diff --git a/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.jpg b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.jpg new file mode 100644 index 00000000000..d264e8a18aa --- /dev/null +++ b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19169c7496ca7e24ca07ea968be245551abd17b9286ea1cc3693237ddabe93fe +size 27761 diff --git a/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.png b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.png new file mode 100644 index 00000000000..47a77d9d17f --- /dev/null +++ b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_11_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df277510a31c59e8b5a372faac4307a5d0190197d1414812d59fd2016dbec5d5 +size 411284 diff --git a/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.jpg b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.jpg new file mode 100644 index 00000000000..bfb87dd12df --- /dev/null +++ b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cac681a4943e89f0f0e94a68b31795869f8b925fe773848eb093e90c7d062b7 +size 118423 diff --git a/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.png b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.png new file mode 100644 index 00000000000..ad1c32f0b63 --- /dev/null +++ b/docs/notebooks/248-ssd-b1-with-output_files/248-ssd-b1-with-output_9_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ed72e50f145826d78a2a5af3c6e4bf2e343cd3a0f1c14ba0f3ec9784e94c48f +size 1700640 diff --git a/docs/notebooks/248-ssd-b1-with-output_files/index.html b/docs/notebooks/248-ssd-b1-with-output_files/index.html new file mode 100644 index 00000000000..085e853cbb8 --- /dev/null +++ b/docs/notebooks/248-ssd-b1-with-output_files/index.html @@ -0,0 +1,10 @@ + +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/248-ssd-b1-with-output_files/ + +

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/248-ssd-b1-with-output_files/


../
+248-ssd-b1-with-output_11_1.jpg                    07-Dec-2023 00:49               27761
+248-ssd-b1-with-output_11_1.png                    07-Dec-2023 00:49              411284
+248-ssd-b1-with-output_9_3.jpg                     07-Dec-2023 00:49              118423
+248-ssd-b1-with-output_9_3.png                     07-Dec-2023 00:49             1700640
+

+ diff --git a/docs/notebooks/248-stable-diffusion-xl-with-output.rst b/docs/notebooks/248-stable-diffusion-xl-with-output.rst index f868ad02114..34504d0646c 100644 --- a/docs/notebooks/248-stable-diffusion-xl-with-output.rst +++ b/docs/notebooks/248-stable-diffusion-xl-with-output.rst @@ -68,7 +68,7 @@ The tutorial consists of the following steps: **Table of contents:** -- `Install Prerequisites <#install-prerequisites>`__ +- `Install prerequisites <#install-prerequisites>`__ - `SDXL Base model <#sdxl-base-model>`__ - `Select inference device SDXL Base @@ -79,28 +79,34 @@ The tutorial consists of the following steps: Demo <#textimage-generation-interactive-demo>`__ - `Run Image2Image generation pipeline <#run-imageimage-generation-pipeline>`__ + + - `Select inference device SDXL Refiner + model <#select-inference-device-sdxl-refiner-model>`__ + - `Image2Image Generation Interactive Demo <#imageimage-generation-interactive-demo>`__ - `SDXL Refiner model <#sdxl-refiner-model>`__ - - `Select inference device SDXL Refiner - model <#select-inference-device-sdxl-refiner-model>`__ + - `Select inference device <#select-inference-device>`__ - `Run Text2Image generation with Refinement <#run-textimage-generation-with-refinement>`__ -Install prerequisites ---------------------------------------------------------------- +Install prerequisites +--------------------- + + .. code:: ipython3 + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "diffusers>=0.18.0" "invisible-watermark>=0.2.0" "transformers>=4.33.0" "accelerate" "onnx" %pip install -q "git+https://github.com/huggingface/optimum-intel.git" - %pip install -q "openvino>=2023.1.0" - %pip install -q --upgrade-strategy eager "diffusers>=0.18.0" "invisible-watermark>=0.2.0" "transformers>=4.30.2" "accelerate" "onnx" "onnxruntime" - %pip install -q gradio + %pip install -q "openvino>=2023.1.0" gradio + +SDXL Base model +--------------- + -SDXL Base model ---------------------------------------------------------- We will start with the base model part, which is responsible for the generation of images of the desired output size. @@ -144,8 +150,10 @@ You can save the model on disk using the ``save_pretrained`` method. warnings.warn( -Select inference device SDXL Base model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device SDXL Base model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -194,8 +202,10 @@ select device from dropdown list for running inference using OpenVINO Compiling the text_encoder_2... -Run Text2Image generation pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Text2Image generation pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Now, we can run the model for the generation of images using text prompts. To speed up evaluation and reduce the required memory we @@ -240,8 +250,10 @@ numpy random state with a specific seed for results reproducibility. -Text2image Generation Interactive Demo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text2image Generation Interactive Demo +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -294,6 +306,7 @@ Text2image Generation Interactive Demo ..
+ .. code:: ipython3 demo.close() @@ -306,16 +319,20 @@ Text2image Generation Interactive Demo Closing server running on port: 7860 -Run Image2Image generation pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Image2Image generation pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + We can reuse the already converted model for running the Image2Image generation pipeline. For that, we should replace ``OVStableDiffusionXLPipeline`` with ``OVStableDiffusionXLImage2ImagePipeline``. -Select inference device SDXL Refiner model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Select inference device SDXL Refiner model +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + select device from dropdown list for running inference using OpenVINO @@ -384,8 +401,10 @@ select device from dropdown list for running inference using OpenVINO -Image2Image Generation Interactive Demo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image2Image Generation Interactive Demo +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -463,8 +482,10 @@ Image2Image Generation Interactive Demo -SDXL Refiner model ------------------------------------------------------------- +SDXL Refiner model +------------------ + + As we discussed above, Stable Diffusion XL can be used in a 2-stages approach: first, the base model is used to generate latents of the @@ -493,8 +514,10 @@ prompt for improving generated image. del refiner gc.collect() -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -511,8 +534,10 @@ select device from dropdown list for running inference using OpenVINO -Run Text2Image generation with Refinement -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Text2Image generation with Refinement +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 diff --git a/docs/notebooks/248-stable-diffusion-xl-with-output_files/index.html b/docs/notebooks/248-stable-diffusion-xl-with-output_files/index.html index 45534552f3d..c514cf17997 100644 --- a/docs/notebooks/248-stable-diffusion-xl-with-output_files/index.html +++ b/docs/notebooks/248-stable-diffusion-xl-with-output_files/index.html @@ -1,12 +1,12 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/248-stable-diffusion-xl-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/248-stable-diffusion-xl-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/248-stable-diffusion-xl-with-output_files/


../
-248-stable-diffusion-xl-with-output_10_3.jpg       31-Oct-2023 00:35               21574
-248-stable-diffusion-xl-with-output_10_3.png       31-Oct-2023 00:35              440317
-248-stable-diffusion-xl-with-output_18_3.jpg       31-Oct-2023 00:35               22767
-248-stable-diffusion-xl-with-output_18_3.png       31-Oct-2023 00:35              439143
-248-stable-diffusion-xl-with-output_29_2.jpg       31-Oct-2023 00:35               29349
-248-stable-diffusion-xl-with-output_29_2.png       31-Oct-2023 00:35              432689
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/248-stable-diffusion-xl-with-output_files/


../
+248-stable-diffusion-xl-with-output_10_3.jpg       07-Dec-2023 00:49               21574
+248-stable-diffusion-xl-with-output_10_3.png       07-Dec-2023 00:49              440317
+248-stable-diffusion-xl-with-output_18_3.jpg       07-Dec-2023 00:49               22767
+248-stable-diffusion-xl-with-output_18_3.png       07-Dec-2023 00:49              439143
+248-stable-diffusion-xl-with-output_29_2.jpg       07-Dec-2023 00:49               29349
+248-stable-diffusion-xl-with-output_29_2.png       07-Dec-2023 00:49              432689
 

diff --git a/docs/notebooks/250-music-generation-with-output.rst b/docs/notebooks/250-music-generation-with-output.rst index 181a0111215..272c18651bb 100644 --- a/docs/notebooks/250-music-generation-with-output.rst +++ b/docs/notebooks/250-music-generation-with-output.rst @@ -32,6 +32,7 @@ library. **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Install requirements <#install-requirements>`__ @@ -72,8 +73,7 @@ Install requirements .. code:: ipython3 %pip install -q "openvino>=2023.1.0" - %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch onnx gradio ipywidgets - %pip install -q "transformers" + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch onnx gradio ipywidgets transformers .. parsed-literal:: @@ -82,8 +82,6 @@ Install requirements Note: you may need to restart the kernel to use updated packages. DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - Note: you may need to restart the kernel to use updated packages. Imports @@ -114,10 +112,10 @@ Imports .. parsed-literal:: - 2023-11-15 00:09:21.886779: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-11-15 00:09:21.920564: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-07 00:16:19.977472: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-07 00:16:20.011221: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-11-15 00:09:22.467173: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-07 00:16:20.555535: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT MusicGen in HF Transformers @@ -191,7 +189,7 @@ vocabulary. It helps the model understand the context of a sentence. @@ -401,13 +399,13 @@ wrapper class with its ``forward()`` method calling .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/symbolic_opset9.py:4315: UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable length with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/symbolic_opset9.py:4315: UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable length with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model. warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) _C._jit_pass_onnx_node_shape_type_inference(node, params_dict, opset_version) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) _C._jit_pass_onnx_graph_shape_type_inference( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.) _C._jit_pass_onnx_graph_shape_type_inference( @@ -643,7 +641,7 @@ We can now infer the pipeline backed by OpenVINO models. diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output.rst b/docs/notebooks/251-tiny-sd-image-generation-with-output.rst index 083da6fe580..7589f656b5f 100644 --- a/docs/notebooks/251-tiny-sd-image-generation-with-output.rst +++ b/docs/notebooks/251-tiny-sd-image-generation-with-output.rst @@ -43,9 +43,8 @@ The notebook contains the following steps: - `Prerequisites <#prerequisites>`__ -- `Create PyTorch Models - pipeline <#create-pytorch-models-pipeline>`__ -- `Convert models to OpenVINO Intermediate representation (IR) +- `Create PyTorch Models pipeline <#create-pytorch-models-pipeline>`__ +- `Convert models to OpenVINO Intermediate representation format <#convert-models-to-openvino-intermediate-representation-format>`__ - `Text Encoder <#text-encoder>`__ @@ -53,26 +52,27 @@ The notebook contains the following steps: - `VAE <#vae>`__ - `Prepare Inference Pipeline <#prepare-inference-pipeline>`__ -- `Configure Inference - Pipeline <#configure-inference-pipeline>`__ +- `Configure Inference Pipeline <#configure-inference-pipeline>`__ - `Text-to-Image generation <#text-to-image-generation>`__ - `Image-to-Image generation <#image-to-image-generation>`__ + - `Interactive Demo <#interactive-demo>`__ + +Prerequisites +------------- -- `Interactive Demo <#interactive-demo>`__ -Prerequisites -------------------------------------------------------- Install required dependencies .. code:: ipython3 - %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision - %pip -q install "openvino>=2023.1.0" "diffusers>=0.18.0" "transformers>=4.30.2" "gradio" + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision "openvino>=2023.1.0" "diffusers>=0.18.0" "transformers>=4.30.2" "gradio" + +Create PyTorch Models pipeline +------------------------------ + -Create PyTorch Models pipeline ------------------------------------------------------------------------- ``StableDiffusionPipeline`` is an end-to-end inference pipeline that you can use to generate images from text with just a few lines of code. @@ -121,8 +121,10 @@ First, load the pre-trained weights of all components of the model. -Convert models to OpenVINO Intermediate representation format -------------------------------------------------------------------------------------------------------- +Convert models to OpenVINO Intermediate representation format +------------------------------------------------------------- + + OpenVINO supports PyTorch through conversion to OpenVINO Intermediate Representation (IR) format. To take the advantage of OpenVINO @@ -150,8 +152,10 @@ The model consists of three important parts: Let us convert each part. -Text Encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text Encoder +~~~~~~~~~~~~ + + The text-encoder is responsible for transforming the input prompt, for example, “a photo of an astronaut riding a horse” into an embedding @@ -220,8 +224,10 @@ hidden states. -U-net -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +U-net +~~~~~ + + U-net model has three inputs: @@ -297,8 +303,10 @@ Model predicts the ``sample`` state for the next step. -VAE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +VAE +~~~ + + The VAE model has two parts, an encoder and a decoder. The encoder is used to convert the image into a low dimensional latent representation, @@ -409,8 +417,10 @@ of the pipeline, it will be better to convert them to separate models. -Prepare Inference Pipeline --------------------------------------------------------------------- +Prepare Inference Pipeline +-------------------------- + + Putting it all together, let us now take a closer look at how the model works in inference by illustrating the logical flow. @@ -810,8 +820,10 @@ of the variational auto encoder. return timesteps, num_inference_steps - t_start -Configure Inference Pipeline ----------------------------------------------------------------------- +Configure Inference Pipeline +---------------------------- + + First, you should create instances of OpenVINO Model. @@ -882,8 +894,10 @@ Let us define them and put all components together scheduler=lms ) -Text-to-Image generation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text-to-Image generation +~~~~~~~~~~~~~~~~~~~~~~~~ + + Now, let’s see model in action @@ -951,8 +965,10 @@ Now is show time! Nice. As you can see, the picture has quite a high definition 🔥. -Image-to-Image generation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Image-to-Image generation +~~~~~~~~~~~~~~~~~~~~~~~~~ + + One of the most amazing features of Stable Diffusion model is the ability to condition image generation from an existing image or sketch. @@ -1052,8 +1068,10 @@ found in this .. image:: 251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation-with-output_39_1.png -Interactive Demo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Interactive Demo +~~~~~~~~~~~~~~~~ + + .. code:: ipython3 diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html index 98e68a3c493..395cb0490dd 100644 --- a/docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html +++ b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html @@ -1,12 +1,12 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/251-tiny-sd-image-generation-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/251-tiny-sd-image-generation-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/251-tiny-sd-image-generation-with-output_files/


../
-251-tiny-sd-image-generation-with-output_33_1.jpg  31-Oct-2023 00:35               40294
-251-tiny-sd-image-generation-with-output_33_1.png  31-Oct-2023 00:35              434441
-251-tiny-sd-image-generation-with-output_37_1.jpg  31-Oct-2023 00:35               84339
-251-tiny-sd-image-generation-with-output_37_1.png  31-Oct-2023 00:35              770190
-251-tiny-sd-image-generation-with-output_39_1.jpg  31-Oct-2023 00:35               50437
-251-tiny-sd-image-generation-with-output_39_1.png  31-Oct-2023 00:35              699062
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/251-tiny-sd-image-generation-with-output_files/


../
+251-tiny-sd-image-generation-with-output_33_1.jpg  07-Dec-2023 00:49               40294
+251-tiny-sd-image-generation-with-output_33_1.png  07-Dec-2023 00:49              434441
+251-tiny-sd-image-generation-with-output_37_1.jpg  07-Dec-2023 00:49               84339
+251-tiny-sd-image-generation-with-output_37_1.png  07-Dec-2023 00:49              770190
+251-tiny-sd-image-generation-with-output_39_1.jpg  07-Dec-2023 00:49               50437
+251-tiny-sd-image-generation-with-output_39_1.png  07-Dec-2023 00:49              699062
 

diff --git a/docs/notebooks/254-llm-chatbot-with-output.rst b/docs/notebooks/254-llm-chatbot-with-output.rst index 78d6200e20d..77e30616a64 100644 --- a/docs/notebooks/254-llm-chatbot-with-output.rst +++ b/docs/notebooks/254-llm-chatbot-with-output.rst @@ -18,7 +18,7 @@ accuracy. Previously, we already discussed how to build an instruction-following pipeline using OpenVINO and Optimum Intel, please check out `Dolly -example <240-dolly-2-instruction-following-with-output.html>`__ for reference. In this +example <../240-dolly-2-instruction-following>`__ for reference. In this tutorial, we consider how to use the power of OpenVINO for running Large Language Models for chat. We will use a pre-trained model from the `Hugging Face @@ -40,16 +40,22 @@ The tutorial consists of the following steps: **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Select model for inference <#select-model-for-inference>`__ -- `login to huggingfacehub to get access to pretrained model <#login-to-huggingfacehub-to-get-access-to-pretrained-model>`__ -- `Instantiate Model using Optimum Intel <#instantiate-model-using-optimum-intel>`__ +- `login to huggingfacehub to get access to pretrained + model <#login-to-huggingfacehub-to-get-access-to-pretrained-model>`__ +- `Instantiate Model using Optimum + Intel <#instantiate-model-using-optimum-intel>`__ - `Compress model weights <#compress-model-weights>`__ - - `Weights Compression using Optimum Intel <#weights-compression-using-optimum-intel>`__ - - `Weights Compression using NNCF <#weights-compression-using-nncf>`__ + - `Weights Compression using Optimum + Intel <#weights-compression-using-optimum-intel>`__ + - `Weights Compression using + NNCF <#weights-compression-using-nncf>`__ -- `Select device for inference and model variant <#select-device-for-inference-and-model-variant>`__ +- `Select device for inference and model + variant <#select-device-for-inference-and-model-variant>`__ - `Run Chatbot <#run-chatbot>`__ Prerequisites @@ -62,12 +68,12 @@ Install required dependencies .. code:: ipython3 %pip uninstall -q -y openvino-dev openvino openvino-nightly - %pip install -q openvino-nightly %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu\ "git+https://github.com/huggingface/optimum-intel.git"\ - "git+https://github.com/openvinotoolkit/nncf.git@release_v270"\ + "nncf>=2.7"\ + "openvino-nightly"\ "gradio"\ - "onnx" "einops" "transformers>=4.34.0"\ + "onnx" "einops" "transformers_stream_generator" "tiktoken" "transformers>=4.34.0" Select model for inference -------------------------- @@ -81,6 +87,18 @@ from user side and at least 64GB RAM for conversion. The available options are: +- **tiny-llama-1b-chat** - This is the chat model finetuned on top of + `TinyLlama/TinyLlama-1.1B-intermediate-step-955k-2T `__. + The TinyLlama project aims to pretrain a 1.1B Llama model on 3 + trillion tokens with the adoption of the same architecture and + tokenizer as Llama 2. This means TinyLlama can be plugged and played + in many open-source projects built upon Llama. Besides, TinyLlama is + compact with only 1.1B parameters. This compactness allows it to + cater to a multitude of applications demanding a restricted + computation and memory footprint. More details about model can be + found in `model + card `__ + - **red-pajama-3b-chat** - A 2.8B parameter pre-trained language model based on GPT-NEOX architecture. It was developed by Together Computer and leaders from the open-source AI community. The model is @@ -109,10 +127,8 @@ The available options are: following code: .. code:: python - :force: ## login to huggingfacehub to get access to pretrained model - from huggingface_hub import notebook_login, whoami try: @@ -142,6 +158,23 @@ The available options are: `repository `__ and `HuggingFace model card `__. +- **qwen-7b-chat** - Qwen-7B is the 7B-parameter version of the large + language model series, Qwen (abbr. Tongyi Qianwen), proposed by + Alibaba Cloud. Qwen-7B is a Transformer-based large language model, + which is pretrained on a large volume of data, including web texts, + books, codes, etc. For more details about Qwen, please refer to the + `GitHub `__ code repository. +- **chatglm2-6b** - ChatGLM2-6B is the second-generation version of the + open-source bilingual (Chinese-English) chat model + `ChatGLM-6B `__. It retains the + smooth conversation flow and low deployment threshold of the + first-generation model +- **mistral-7b** - The Mistral-7B-v0.1 Large Language Model (LLM) is a + pretrained generative text model with 7 billion parameters. You can + find more details about model in the `model + card `__, + `paper `__ and `release blog + post `__. - **zephyr-7b-beta** - Zephyr is a series of language models that are trained to act as helpful assistants. Zephyr-7B-beta is the second model in the series, and is a fine-tuned version of @@ -152,6 +185,26 @@ The available options are: details about model in `technical report `__ and `HuggingFace model card `__. +- **neural-chat-7b-v3-1** - Mistral-7b model fine-tuned using Intel + Gaudi. The model fine-tuned on the open source dataset + `Open-Orca/SlimOrca `__ + and aligned with `Direct Preference Optimization (DPO) + algorithm `__. More details can be + found in `model + card `__ and `blog + post `__. +- **notus-7b-v1** - Notus is a collection of fine-tuned models using + `Direct Preference Optimization + (DPO) `__. and related + `RLHF `__ techniques. This model is + the first version, fine-tuned with DPO over zephyr-7b-sft. Following + a data-first approach, the only difference between Notus-7B-v1 and + Zephyr-7B-beta is the preference dataset used for dDPO. Proposed + approach for dataset creation helps to effectively fine-tune Notus-7b + that surpasses Zephyr-7B-beta and Claude 2 on + `AlpacaEval `__. More + details about model can be found in `model + card `__. .. code:: ipython3 @@ -164,8 +217,8 @@ The available options are: model_id = widgets.Dropdown( options=model_ids, - value=model_ids[-1], - description='Model:', + value=model_ids[0], + description="Model:", disabled=False, ) @@ -176,7 +229,7 @@ The available options are: .. parsed-literal:: - Dropdown(description='Model:', index=3, options=('red-pajama-3b-chat', 'llama-2-chat-7b', 'mpt-7b-chat', 'zeph… + Dropdown(description='Model:', options=('tiny-llama-1b-chat', 'red-pajama-3b-chat', 'llama-2-chat-7b', 'mpt-7b… @@ -188,7 +241,7 @@ The available options are: .. parsed-literal:: - Selected model zephyr-7b-beta + Selected model tiny-llama-1b-chat Instantiate Model using Optimum Intel @@ -240,99 +293,23 @@ hidden states for the current step as output. It means for all next iterations, it is enough to provide only a new token obtained from the previous step and cached key values to get the next token prediction. -In our case, MPT model currently is not covered by Optimum Intel, we -will convert it manually and create wrapper compatible with Optimum -Intel. - -Below is some code required for MPT conversion. +In our case, MPT, Qwen and ChatGLM model currently is not covered by +Optimum Intel, we will convert it manually and create wrapper compatible +with Optimum Intel. .. code:: ipython3 - from functools import wraps - import torch - from transformers import AutoModelForCausalLM - from nncf import compress_weights + from transformers import AutoModelForCausalLM, AutoConfig + from optimum.intel import OVQuantizer + from optimum.intel.openvino import OVModelForCausalLM import openvino as ov from pathlib import Path - from typing import Optional, Union, Dict, Tuple, List - - def flattenize_inputs(inputs): - """ - Helper function for making nested inputs flattens - """ - flatten_inputs = [] - for input_data in inputs: - if input_data is None: - continue - if isinstance(input_data, (list, tuple)): - flatten_inputs.extend(flattenize_inputs(input_data)) - else: - flatten_inputs.append(input_data) - return flatten_inputs - - def cleanup_torchscript_cache(): - """ - Helper for removing cached model representation - """ - torch._C._jit_clear_class_registry() - torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() - torch.jit._state._clear_class_state() - - def convert_mpt(pt_model:torch.nn.Module, model_path:Path): - """ - MPT model conversion function - - Params: - pt_model: PyTorch model - model_path: path for saving model - Returns: - None - """ - ov_out_path = Path(model_path) / "openvino_model.xml" - pt_model.config.save_pretrained(ov_out_path.parent) - pt_model.config.use_cache = True - outs = pt_model(input_ids=torch.ones((1, 10), dtype=torch.long), attention_mask=torch.ones((1, 10), dtype=torch.long)) - inputs = ["input_ids"] - outputs = ["logits"] - - dynamic_shapes = {"input_ids": {1: "seq_len"}, "attention_mask": {1: "seq_len"}} - for idx in range(len(outs.past_key_values)): - inputs.extend([f"past_key_values.{idx}.key", f"past_key_values.{idx}.value"]) - dynamic_shapes[inputs[-1]] = {2: "past_sequence + sequence"} - dynamic_shapes[inputs[-2]] = {3: "past_sequence + sequence"} - outputs.extend([f"present.{idx}.key", f"present.{idx}.value"]) - - inputs.append("attention_mask") - dummy_inputs = {"input_ids": torch.ones((1,2), dtype=torch.long), "past_key_values": outs.past_key_values, "attention_mask": torch.ones((1,12), dtype=torch.long)} - pt_model.config.torchscript = True - orig_forward = pt_model.forward - @wraps(orig_forward) - def ts_patched_forward(input_ids: torch.Tensor, past_key_values: Tuple[Tuple[torch.Tensor]], attention_mask: torch.Tensor): - pkv_list = list(past_key_values) - outs = orig_forward(input_ids=input_ids, past_key_values=pkv_list, attention_mask=attention_mask) - return (outs.logits, tuple(outs.past_key_values)) - pt_model.forward = ts_patched_forward - ov_model = ov.convert_model(pt_model, example_input=dummy_inputs) - pt_model.forward = orig_forward - for inp_name, m_input, input_data in zip(inputs, ov_model.inputs, flattenize_inputs(dummy_inputs.values())): - input_node = m_input.get_node() - if input_node.element_type == ov.Type.dynamic: - m_input.get_node().set_element_type(ov.Type.f32) - shape = list(input_data.shape) - if inp_name in dynamic_shapes: - for k in dynamic_shapes[inp_name]: - shape[k] = -1 - input_node.set_partial_shape(ov.PartialShape(shape)) - m_input.get_tensor().set_names({inp_name}) - - for out, out_name in zip(ov_model.outputs, outputs): - out.get_tensor().set_names({out_name}) - - ov_model.validate_nodes_and_infer_types() - ov.save_model(ov_model, ov_out_path) - del ov_model - cleanup_torchscript_cache() - del pt_model + import shutil + import torch + import logging + import nncf + import gc + from converter import converters .. parsed-literal:: @@ -340,8 +317,13 @@ Below is some code required for MPT conversion. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino -Compress model weights ----------------------------------------------------------------- +.. parsed-literal:: + + No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' + + +Compress model weights +---------------------- @@ -353,8 +335,8 @@ larger than the size of activations, for example, Large Language Models performance even more, but introduces a minor drop in prediction quality. -Weights Compression using Optimum Intel -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Weights Compression using Optimum Intel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -374,8 +356,8 @@ LLAMA and Zephyr examples. **Note**: There may be no speedup for INT4/INT8 compressed models on dGPU. -Weights Compression using NNCF -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Weights Compression using NNCF +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -393,22 +375,20 @@ will consider this variant based on MPT model. from IPython.display import display - # TODO: red-pajama-3b-chat currently can't be compiled in INT4 or FP16 due to ticket 123973 - is_pajama_model = model_id.value == 'red-pajama-3b-chat' prepare_int4_model = widgets.Checkbox( - value=True and not is_pajama_model, - description='Prepare INT4 model', - disabled=is_pajama_model, + value=True, + description="Prepare INT4 model", + disabled=False, ) prepare_int8_model = widgets.Checkbox( - value=False or is_pajama_model, - description='Prepare INT8 model', + value=False, + description="Prepare INT8 model", disabled=False, ) prepare_fp16_model = widgets.Checkbox( value=False, - description='Prepare FP16 model', - disabled=is_pajama_model, + description="Prepare FP16 model", + disabled=False, ) display(prepare_int4_model) @@ -438,138 +418,231 @@ We can now save floating point and compressed model variants .. code:: ipython3 - from pathlib import Path - from optimum.intel import OVQuantizer - from optimum.intel.openvino import OVModelForCausalLM - import shutil - import logging - import nncf - import gc - nncf.set_log_level(logging.ERROR) pt_model_id = model_configuration["model_id"] + pt_model_name = model_id.value.split("-")[0] + model_type = AutoConfig.from_pretrained(pt_model_id, trust_remote_code=True).model_type fp16_model_dir = Path(model_id.value) / "FP16" int8_model_dir = Path(model_id.value) / "INT8_compressed_weights" int4_model_dir = Path(model_id.value) / "INT4_compressed_weights" + def convert_to_fp16(): if (fp16_model_dir / "openvino_model.xml").exists(): return - if "mpt" not in model_id.value: - ov_model = OVModelForCausalLM.from_pretrained(pt_model_id, export=True, compile=False) + if not model_configuration["remote"]: + ov_model = OVModelForCausalLM.from_pretrained( + pt_model_id, export=True, compile=False + ) ov_model.half() ov_model.save_pretrained(fp16_model_dir) del ov_model else: - model = AutoModelForCausalLM.from_pretrained(model_configuration["model_id"], torch_dtype=torch.float32, trust_remote_code=True) - convert_mpt(model, fp16_model_dir) + model_kwargs = {} + if "revision" in model_configuration: + model_kwargs["revision"] = model_configuration["revision"] + model = AutoModelForCausalLM.from_pretrained( + model_configuration["model_id"], + torch_dtype=torch.float32, + trust_remote_code=True, + **model_kwargs + ) + converters[pt_model_name](model, fp16_model_dir) del model gc.collect() + def convert_to_int8(): if (int8_model_dir / "openvino_model.xml").exists(): return - if "mpt" not in model_id.value: - if not fp16_model_dir.exists(): - ov_model = OVModelForCausalLM.from_pretrained(pt_model_id, export=True, compile=False) - ov_model.half() - else: + int8_model_dir.mkdir(parents=True, exist_ok=True) + if not model_configuration["remote"]: + if fp16_model_dir.exists(): ov_model = OVModelForCausalLM.from_pretrained(fp16_model_dir, compile=False) + else: + ov_model = OVModelForCausalLM.from_pretrained( + pt_model_id, export=True, compile=False + ) + ov_model.half() quantizer = OVQuantizer.from_pretrained(ov_model) quantizer.quantize(save_directory=int8_model_dir, weights_only=True) del quantizer del ov_model else: convert_to_fp16() - model = ov.Core().read_model(fp16_model_dir / 'openvino_model.xml') - compressed_model = compress_weights(model) + ov_model = ov.Core().read_model(fp16_model_dir / "openvino_model.xml") + shutil.copy(fp16_model_dir / "config.json", int8_model_dir / "config.json") + configuration_file = fp16_model_dir / f"configuration_{model_type}.py" + if configuration_file.exists(): + shutil.copy( + configuration_file, int8_model_dir / f"configuration_{model_type}.py" + ) + compressed_model = nncf.compress_weights(ov_model) ov.save_model(compressed_model, int8_model_dir / "openvino_model.xml") - shutil.copy(fp16_model_dir / 'config.json', int8_model_dir / 'config.json') - del model + del ov_model del compressed_model gc.collect() - def convert_to_int4(group_size, ratio): - if (int4_model_dir / "openvino_model").exists(): + def convert_to_int4(): + compression_configs = { + "zephyr-7b-beta": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 64, + "ratio": 0.6, + }, + "mistral-7b": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 64, + "ratio": 0.6, + }, + "notus-7b-v1": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 64, + "ratio": 0.6, + }, + "neural-chat-7b-v3-1": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 64, + "ratio": 0.6, + }, + "llama-2-chat-7b": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 128, + "ratio": 0.8, + }, + "chatglm2-6b": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 128, + "ratio": 0.72, + "ignored_scope": nncf.IgnoredScope(["__module.transformer/aten::index_67/Gather"]) + }, + "qwen-7b-chat": { + "mode": nncf.CompressWeightsMode.INT4_SYM, + "group_size": 128, + "ratio": 0.6 + }, + 'red-pajama-3b-chat': { + "mode": nncf.CompressWeightsMode.INT4_ASYM, + "group_size": 128, + "ratio": 0.5, + }, + "default": { + "mode": nncf.CompressWeightsMode.INT4_ASYM, + "group_size": 128, + "ratio": 0.8, + }, + } + + model_compression_params = compression_configs.get( + model_id.value, compression_configs["default"] + ) + if (int4_model_dir / "openvino_model.xml").exists(): return int4_model_dir.mkdir(parents=True, exist_ok=True) - if "mpt" not in model_id.value: - # TODO: remove compression via NNCF for non-MPT models when INT4 weight compression is added to optimum-intel + if not model_configuration["remote"]: if not fp16_model_dir.exists(): - model = OVModelForCausalLM.from_pretrained(pt_model_id, export=True, compile=False) - model.half() + model = OVModelForCausalLM.from_pretrained( + pt_model_id, export=True, compile=False + ).half() + model.config.save_pretrained(int4_model_dir) + ov_model = model.model + del model + gc.collect() else: - model = OVModelForCausalLM.from_pretrained(fp16_model_dir, compile=False) - model.config.save_pretrained(int4_model_dir) - ov_model = model.model - del model + ov_model = ov.Core().read_model(fp16_model_dir / "openvino_model.xml") + shutil.copy(fp16_model_dir / "config.json", int4_model_dir / "config.json") + else: convert_to_fp16() - ov_model = ov.Core().read_model(fp16_model_dir / 'openvino_model.xml') - shutil.copy(fp16_model_dir / 'config.json', int4_model_dir / 'config.json') - compressed_model = nncf.compress_weights(ov_model, mode=nncf.CompressWeightsMode.INT4_ASYM, group_size=group_size, ratio=ratio) - ov.save_model(compressed_model, int4_model_dir / 'openvino_model.xml') + ov_model = ov.Core().read_model(fp16_model_dir / "openvino_model.xml") + shutil.copy(fp16_model_dir / "config.json", int4_model_dir / "config.json") + configuration_file = fp16_model_dir / f"configuration_{model_type}.py" + if configuration_file.exists(): + shutil.copy( + configuration_file, int4_model_dir / f"configuration_{model_type}.py" + ) + compressed_model = nncf.compress_weights(ov_model, **model_compression_params) + ov.save_model(compressed_model, int4_model_dir / "openvino_model.xml") del ov_model del compressed_model gc.collect() + if prepare_fp16_model.value: - print("Apply weights compression to FP16 format") convert_to_fp16() if prepare_int8_model.value: - print("Apply weights compression to INT8 format") convert_to_int8() if prepare_int4_model.value: - print("Apply weights compression to INT4 format") - convert_to_int4(group_size=128, ratio=0.8) + convert_to_int4() + .. parsed-literal:: - No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' + config.json: 0%| | 0.00/699 [00:00 True - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/transformers/models/mistral/modeling_mistral.py:795: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if input_shape[-1] > 1: - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/transformers/models/mistral/modeling_mistral.py:91: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /home/ea/work/genai_env/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:94: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if (input_shape[-1] > 1 or self.sliding_window is not None) and self.is_causal: + /home/ea/work/genai_env/lib/python3.8/site-packages/optimum/exporters/onnx/model_patcher.py:392: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/transformers/models/mistral/modeling_mistral.py:157: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /home/ea/work/genai_env/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:140: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.max_seq_len_cached: - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/transformers/models/mistral/modeling_mistral.py:288: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /home/ea/work/genai_env/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:392: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/transformers/models/mistral/modeling_mistral.py:295: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /home/ea/work/genai_env/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:399: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/transformers/models/mistral/modeling_mistral.py:306: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /home/ea/work/genai_env/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:409: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): @@ -624,21 +697,25 @@ Let’s compare model size for different compression types int4_weights = int4_model_dir / "openvino_model.bin" if fp16_weights.exists(): - print(f'Size of FP16 model is {fp16_weights.stat().st_size / 1024 / 1024:.2f} MB') + print(f"Size of FP16 model is {fp16_weights.stat().st_size / 1024 / 1024:.2f} MB") for precision, compressed_weights in zip([8, 4], [int8_weights, int4_weights]): if compressed_weights.exists(): - print(f'Size of model with INT{precision} compressed weights is {compressed_weights.stat().st_size / 1024 / 1024:.2f} MB') + print( + f"Size of model with INT{precision} compressed weights is {compressed_weights.stat().st_size / 1024 / 1024:.2f} MB" + ) if compressed_weights.exists() and fp16_weights.exists(): - print(f"Compression rate for INT{precision} model: {fp16_weights.stat().st_size / compressed_weights.stat().st_size:.3f}") + print( + f"Compression rate for INT{precision} model: {fp16_weights.stat().st_size / compressed_weights.stat().st_size:.3f}" + ) .. parsed-literal:: - Size of model with INT4 compressed weights is 4374.50 MB + Size of model with INT4 compressed weights is 696.99 MB -Select device for inference and model variant ---------------------------------------------------------------------------------------- +Select device for inference and model variant +--------------------------------------------- @@ -650,8 +727,8 @@ Select device for inference and model variant core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], - value='CPU', - description='Device:', + value="CPU", + description="Device:", disabled=False, ) @@ -662,160 +739,16 @@ Select device for inference and model variant .. parsed-literal:: - Dropdown(description='Device:', options=('CPU', 'GPU', 'AUTO'), value='CPU') + Dropdown(description='Device:', options=('CPU', 'GPU.0', 'GPU.1', 'AUTO'), value='CPU') -The cell below create ``OVMPTModel`` model wrapper based on -``OVModelForCausalLM`` model. +The cell below create ``OVMPTModel``, ``OVQWENModel`` and +``OVCHATGLM2Model`` wrapper based on ``OVModelForCausalLM`` model. .. code:: ipython3 - from transformers import AutoConfig, PretrainedConfig - import torch - - from optimum.utils import NormalizedTextConfig, NormalizedConfigManager - from transformers.modeling_outputs import CausalLMOutputWithPast - from optimum.intel.openvino.utils import OV_XML_FILE_NAME - import numpy as np - from pathlib import Path - - - class OVMPTModel(OVModelForCausalLM): - """ - Optimum intel compatible model wrapper for MPT - """ - def __init__( - self, - model: "Model", - config: "PretrainedConfig" = None, - device: str = "CPU", - dynamic_shapes: bool = True, - ov_config: Optional[Dict[str, str]] = None, - model_save_dir: Optional[Union[str, Path]] = None, - **kwargs, - ): - NormalizedConfigManager._conf["mpt"] = NormalizedTextConfig.with_args(num_layers="n_layers", num_attention_heads="n_heads") - super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) - - def _reshape( - self, - model: "Model", - *args, - **kwargs - ): - shapes = {} - for inputs in model.inputs: - shapes[inputs] = inputs.get_partial_shape() - if shapes[inputs].rank.get_length() in [2, 3]: - shapes[inputs][1] = -1 - else: - if ".key" in inputs.get_any_name(): - shapes[inputs][3] = -1 - else: - shapes[inputs][2] = -1 - - model.reshape(shapes) - return model - - def forward( - self, - input_ids: torch.LongTensor, - attention_mask: Optional[torch.LongTensor] = None, - past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, - **kwargs, - ) -> CausalLMOutputWithPast: - self.compile() - - if self.use_cache and past_key_values is not None: - input_ids = input_ids[:, -1:] - - inputs = {} - if past_key_values is not None: - # Flatten the past_key_values - past_key_values = tuple( - past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer - ) - # Add the past_key_values to the decoder inputs - inputs = dict(zip(self.key_value_input_names, past_key_values)) - - # Create empty past_key_values for decoder_with_past first generation step - elif self.use_cache: - shape_input_ids = input_ids.shape - num_attention_heads = ( - self.normalized_config.num_attention_heads if self.config.model_type == "bloom" else 1 - ) - for input_name in self.key_value_input_names: - model_inputs = self.model.input(input_name) - shape = model_inputs.get_partial_shape() - shape[0] = shape_input_ids[0] * num_attention_heads - if shape[2].is_dynamic: - shape[2] = 0 - if shape[1].is_dynamic: - shape[1] = 0 - if shape.rank.get_length() == 4 and shape[3].is_dynamic: - shape[3] = 0 - inputs[input_name] = ov.Tensor(model_inputs.get_element_type(), shape.get_shape()) - - inputs["input_ids"] = np.array(input_ids) - - # Add the attention_mask inputs when needed - if "attention_mask" in self.input_names and attention_mask is not None: - inputs["attention_mask"] = np.array(attention_mask) - - # Run inference - self.request.start_async(inputs, shared_memory=True) - self.request.wait() - - logits = torch.from_numpy(self.request.get_tensor("logits").data).to(self.device) - - if self.use_cache: - # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the self-attention layer) - past_key_values = tuple(self.request.get_tensor(key).data for key in self.key_value_output_names) - # Tuple of tuple of length `n_layers`, with each tuple of length equal to 2 (k/v of self-attention) - past_key_values = tuple( - past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) - ) - else: - past_key_values = None - - return CausalLMOutputWithPast(logits=logits, past_key_values=past_key_values) - - @classmethod - def _from_pretrained( - cls, - model_id: Union[str, Path], - config: PretrainedConfig, - use_auth_token: Optional[Union[bool, str, None]] = None, - revision: Optional[Union[str, None]] = None, - force_download: bool = False, - cache_dir: Optional[str] = None, - file_name: Optional[str] = None, - subfolder: str = "", - from_onnx: bool = False, - local_files_only: bool = False, - load_in_8bit: bool = False, - **kwargs, - ): - model_path = Path(model_id) - default_file_name = OV_XML_FILE_NAME - file_name = file_name or default_file_name - - model_cache_path = cls._cached_file( - model_path=model_path, - use_auth_token=use_auth_token, - revision=revision, - force_download=force_download, - cache_dir=cache_dir, - file_name=file_name, - subfolder=subfolder, - local_files_only=local_files_only, - ) - - model = cls.load_model(model_cache_path, load_in_8bit=load_in_8bit) - init_cls = OVMPTModel - - return init_cls(model=model, config=config, model_save_dir=model_cache_path.parent, **kwargs) + from ovmodel import model_classes The cell below demonstrates how to instantiate model based on selected variant of model weights and inference device @@ -833,8 +766,9 @@ variant of model weights and inference device model_to_run = widgets.Dropdown( options=available_models, value=available_models[0], - description='Model to run:', - disabled=False) + description="Model to run:", + disabled=False, + ) model_to_run @@ -849,8 +783,6 @@ variant of model weights and inference device .. code:: ipython3 - from pathlib import Path - from optimum.intel.openvino import OVModelForCausalLM from transformers import AutoTokenizer if model_to_run.value == "INT4": @@ -860,24 +792,34 @@ variant of model weights and inference device else: model_dir = fp16_model_dir print(f"Loading model from {model_dir}") - model_name = model_configuration["model_id"] - ov_config = {'PERFORMANCE_HINT': 'LATENCY', 'NUM_STREAMS': '1', "CACHE_DIR": ""} + model_name = model_configuration["model_id"] + class_key = model_id.value.split("-")[0] + ov_config = {"PERFORMANCE_HINT": "LATENCY", "NUM_STREAMS": "1", "CACHE_DIR": ""} tok = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - model_class = OVModelForCausalLM if "mpt" not in model_id.value else OVMPTModel - ov_model = model_class.from_pretrained(model_dir, device=device.value, ov_config=ov_config, config=AutoConfig.from_pretrained(model_dir, trust_remote_code=True), trust_remote_code=True) + model_class = ( + OVModelForCausalLM + if not model_configuration["remote"] + else model_classes[class_key] + ) + ov_model = model_class.from_pretrained( + model_dir, + device=device.value, + ov_config=ov_config, + config=AutoConfig.from_pretrained(model_dir, trust_remote_code=True), + trust_remote_code=True, + ) .. parsed-literal:: - Loading model from zephyr-7b-beta/INT4_compressed_weights + Loading model from tiny-llama-1b-chat/INT4_compressed_weights .. parsed-literal:: - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. The argument `trust_remote_code` is to be used along with export=True. It will be ignored. Compiling the model to CPU ... @@ -888,18 +830,19 @@ variant of model weights and inference device test_string = "2 + 2 =" input_tokens = tok(test_string, return_tensors="pt", **tokenizer_kwargs) answer = ov_model.generate(**input_tokens, max_new_tokens=2) - print(tok.batch_decode(answer)[0]) + print(tok.batch_decode(answer, skip_special_tokens=True)[0]) .. parsed-literal:: - /home/ea/work/openvino_notebooks/test_env/lib/python3.8/site-packages/optimum/intel/openvino/modeling_decoder.py:388: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + Setting `pad_token_id` to `eos_token_id`:2 for open-end generation. + /home/ea/work/genai_env/lib/python3.8/site-packages/optimum/intel/openvino/modeling_decoder.py:388: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. self.request.start_async(inputs, shared_memory=True) .. parsed-literal:: - 2 + 2 = 4 + 2 + 2 = 4 Run Chatbot @@ -986,9 +929,8 @@ answers. from threading import Event, Thread from uuid import uuid4 - + from typing import List, Tuple import gradio as gr - import torch from transformers import ( AutoTokenizer, StoppingCriteria, @@ -1004,42 +946,78 @@ answers. stop_tokens = model_configuration.get("stop_tokens") tokenizer_kwargs = model_configuration.get("tokenizer_kwargs", {}) + chinese_examples = [ + ["你好!"], + ["你是谁?"], + ["请介绍一下上海"], + ["请介绍一下英特尔公司"], + ["晚上睡不着怎么办?"], + ["给我讲一个年轻人奋斗创业最终取得成功的故事。"], + ["给这个故事起一个标题。"], + ] + english_examples = [ + ["Hello there! How are you doing?"], + ["What is OpenVINO?"], + ["Who are you?"], + ["Can you explain to me briefly what is Python programming language?"], + ["Explain the plot of Cinderella in a sentence."], + ["What are some common mistakes to avoid when writing code?"], + [ + "Write a 100-word blog post on “Benefits of Artificial Intelligence and OpenVINO“" + ], + ] + + examples = ( + chinese_examples + if ("qwen" in model_id.value or "chatglm" in model_id.value) + else english_examples + ) max_new_tokens = 256 + class StopOnTokens(StoppingCriteria): def __init__(self, token_ids): self.token_ids = token_ids - def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: + + def __call__( + self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs + ) -> bool: for stop_id in self.token_ids: if input_ids[0][-1] == stop_id: return True return False + if stop_tokens is not None: if isinstance(stop_tokens[0], str): stop_tokens = tok.convert_tokens_to_ids(stop_tokens) - + stop_tokens = [StopOnTokens(stop_tokens)] - def default_partial_text_processor(partial_text:str, new_text:str): + + def default_partial_text_processor(partial_text: str, new_text: str): """ helper for updating partially generated answer, used by de - + Params: partial_text: text buffer for storing previosly generated text new_text: text update for the current step Returns: updated text string - + """ partial_text += new_text return partial_text - text_processor = model_configuration.get("partial_text_processor", default_partial_text_processor) - - def convert_history_to_text(history:List[Tuple[str, str]]): + + text_processor = model_configuration.get( + "partial_text_processor", default_partial_text_processor + ) + + + def convert_history_to_text(history: List[Tuple[str, str]]): """ function for conversion history stored as list pairs of user and assistant messages to string according to model expected conversation template Params: @@ -1050,18 +1028,20 @@ answers. text = start_message + "".join( [ "".join( - [ - history_template.format(user=item[0], assistant=item[1]) - ] + [history_template.format(num=round, user=item[0], assistant=item[1])] ) - for item in history[:-1] + for round, item in enumerate(history[:-1]) ] ) text += "".join( [ "".join( [ - current_message_template.format(user=history[-1][0], assistant=history[-1][1]) + current_message_template.format( + num=len(history) + 1, + user=history[-1][0], + assistant=history[-1][1], + ) ] ) ] @@ -1069,11 +1049,10 @@ answers. return text - def user(message, history): """ callback function for updating user messages in interface on submit button click - + Params: message: current message history: conversation history @@ -1087,16 +1066,16 @@ answers. def bot(history, temperature, top_p, top_k, repetition_penalty, conversation_id): """ callback function for running chatbot on submit button click - + Params: history: conversation history - temperature: parameter for control the level of creativity in AI-generated text. + temperature: parameter for control the level of creativity in AI-generated text. By adjusting the `temperature`, you can influence the AI model's probability distribution, making the text more focused or diverse. top_p: parameter for control the range of tokens considered by the AI model based on their cumulative probability. top_k: parameter for control the range of tokens considered by the AI model based on their cumulative probability, selecting number of tokens with highest probability. repetition_penalty: parameter for penalizing tokens based on how frequently they occur in the text. conversation_id: unique conversation identifier. - + """ # Construct the input message string for the model by concatenating the current system message and conversation history @@ -1108,7 +1087,9 @@ answers. history = [history[-1]] messages = convert_history_to_text(history) input_ids = tok(messages, return_tensors="pt", **tokenizer_kwargs).input_ids - streamer = TextIteratorStreamer(tok, timeout=30.0, skip_prompt=True, skip_special_tokens=True) + streamer = TextIteratorStreamer( + tok, timeout=30.0, skip_prompt=True, skip_special_tokens=True + ) generate_kwargs = dict( input_ids=input_ids, max_new_tokens=max_new_tokens, @@ -1155,9 +1136,7 @@ answers. css=".disclaimer {font-variant-caps: all-small-caps;}", ) as demo: conversation_id = gr.State(get_uuid) - gr.Markdown( - f"""

OpenVINO {model_id.value} Chatbot

""" - ) + gr.Markdown(f"""

OpenVINO {model_id.value} Chatbot

""") chatbot = gr.Chatbot(height=500) with gr.Row(): with gr.Column(): @@ -1165,7 +1144,7 @@ answers. label="Chat Message Box", placeholder="Chat Message Box", show_label=False, - container=False + container=False, ) with gr.Column(): with gr.Row(): @@ -1222,17 +1201,8 @@ answers. interactive=True, info="Penalize repetition — 1.0 to disable.", ) - gr.Examples([ - ["Hello there! How are you doing?"], - ["What is OpenVINO?"], - ["Who are you?"], - ["Can you explain to me briefly what is Python programming language?"], - ["Explain the plot of Cinderella in a sentence."], - ["What are some common mistakes to avoid when writing code?"], - ["Write a 100-word blog post on “Benefits of Artificial Intelligence and OpenVINO“"] - ], - inputs=msg, - label="Click on any example and press the 'Submit' button" + gr.Examples( + examples, inputs=msg, label="Click on any example and press the 'Submit' button" ) submit_event = msg.submit( @@ -1288,7 +1258,21 @@ answers. # it creates a publicly shareable link for the interface. Read more in the docs: https://gradio.app/docs/ demo.launch() + +.. parsed-literal:: + + Running on local URL: http://127.0.0.1:7860 + + To create a public link, set `share=True` in `launch()`. + + + +.. .. raw:: html + +..
+ + .. code:: ipython3 # please run this cell for stopping gradio interface - demo.close() + # demo.close() diff --git a/docs/notebooks/256-bark-text-to-audio-with-output.rst b/docs/notebooks/256-bark-text-to-audio-with-output.rst index 4ffede0bda0..3075db69c00 100644 --- a/docs/notebooks/256-bark-text-to-audio-with-output.rst +++ b/docs/notebooks/256-bark-text-to-audio-with-output.rst @@ -58,14 +58,12 @@ is used to act as an audio representation. - `Prerequisites <#prerequisites>`__ -- `Download and Convert - models <#download-and-convert-models>`__ +- `Download and Convert models <#download-and-convert-models>`__ - `Text Encoder <#text-encoder>`__ - `Coarse encoder <#coarse-encoder>`__ - `Fine encoder <#fine-encoder>`__ - - `Prepare Inference - pipeline <#prepare-inference-pipeline>`__ + - `Prepare Inference pipeline <#prepare-inference-pipeline>`__ - `Run model inference <#run-model-inference>`__ @@ -73,22 +71,26 @@ is used to act as an audio representation. - `Interactive demo <#interactive-demo>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- + + .. code:: ipython3 import sys if sys.platform == "linux": - %pip install -q "torch==1.13.1" "torchvision" "torchaudio==0.13.1" --index-url https://download.pytorch.org/whl/cpu + %pip install -q "torch==1.13.1" "torchvision==0.14.1" "torchaudio==0.13.1" --index-url https://download.pytorch.org/whl/cpu else: - %pip install -q "torch==1.13.1" "torchvision" "torchaudio==0.13.1" + %pip install -q "torch==1.13.1" "torchvision==0.14.1" "torchaudio==0.13.1" %pip install -q "openvino>=2023.1.0" gradio %pip install -q "git+https://github.com/suno-ai/bark.git" -Download and Convert models ---------------------------------------------------------------------- +Download and Convert models +--------------------------- + + .. code:: ipython3 @@ -104,8 +106,10 @@ Download and Convert models torch version does not support flash attention. You will get faster inference speed by upgrade torch to newest nightly version. -Text Encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Text Encoder +~~~~~~~~~~~~ + + Text encoder is responsible for embedding initial text prompt into high-level semantic tokens. it uses tokenizer for conversion input text @@ -163,8 +167,10 @@ models for that. del text_encoder_exportable del text_encoder_model, text_encoder -Coarse encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Coarse encoder +~~~~~~~~~~~~~~ + + Coarse encoder is a causal autoregressive transformer, that takes as input the results of the text encoder model. It aims at predicting the @@ -257,8 +263,10 @@ provide empty tensor on the first step. fine_feature_extractor_path = fine_model_dir / "bark_fine_feature_extractor.xml" -Fine encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Fine encoder +~~~~~~~~~~~~ + + Fine encoder is time a non-causal autoencoder transformer, which iteratively predicts the last codebooks based on the sum of the previous @@ -286,8 +294,10 @@ codebooks embeddings obtained using Coarse encoder. fine_model_dir / f"bark_fine_lm_{i}.xml", ) -Prepare Inference pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare Inference pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + For better usability, classes for working with models provided below. @@ -859,14 +869,18 @@ consists from several steps, illustrated on the diagram below: gen_fine_arr = gen_fine_arr[:, :-n_remove_from_end] return gen_fine_arr -Run model inference -------------------------------------------------------------- +Run model inference +------------------- + + Now is time to see model in action. We need only wrap our models to classes and run ``generate_audio`` function. -Select Inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select Inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -952,8 +966,10 @@ select device from dropdown list for running inference using OpenVINO -Interactive demo ----------------------------------------------------------- +Interactive demo +---------------- + + .. code:: ipython3 diff --git a/docs/notebooks/257-llava-multimodal-chatbot-with-output.rst b/docs/notebooks/257-llava-multimodal-chatbot-with-output.rst index 717605194f3..4db0fb97ec0 100644 --- a/docs/notebooks/257-llava-multimodal-chatbot-with-output.rst +++ b/docs/notebooks/257-llava-multimodal-chatbot-with-output.rst @@ -43,6 +43,7 @@ The tutorial consists from following steps: **Table of contents:** + - `About model <#about-model>`__ - `Prerequisites <#prerequisites>`__ - `Build model tokenizer and image @@ -113,7 +114,7 @@ Install required dependencies import sys %pip install -q "torch>=2.1.0" "torchvision" "torchaudio" --index-url https://download.pytorch.org/whl/cpu - %pip install -q "openvino-nightly==2023.2.0.dev20231102" "git+https://github.com/openvinotoolkit/nncf.git@release_v270" "sentencepiece" "tokenizers>=0.12.1" "transformers>=4.31.0,<4.35.0" "gradio" "einops" + %pip install -q "openvino>=2023.2.0" "nncf>=2.7.0" "sentencepiece" "tokenizers>=0.12.1" "transformers>=4.31.0,<4.35.0" "gradio" "einops" .. code:: ipython3 @@ -227,7 +228,7 @@ The code below preparing function for converting LLaVA model to OpenVINO Intermediate Representation format. It splits model on parts described above, prepare example inputs for each part and convert each part using `OpenVINO Model Conversion -API `__. +API `__. ``ov.convert_model`` function accepts PyTorch model instance and returns ``ov.Model`` object that represent model in OpenVINO format. It is ready to use for loading on device using ``ov.compile_model`` or can be saved @@ -484,7 +485,7 @@ Convert model to OpenVINO format and save it on disk. Let’s consider each step more deeply. -Instantiate PyTorch model +Instantiate PyTorch model `:math:`\Uparrow` <#table-of-content>`__ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -495,7 +496,7 @@ from `HuggingFace hub `__ during first run. It may takes some time and requires at least 13 Gb free space on disk. -Compress Model weights to 4 and 8 bits using NNCF +Compress Model weights to 4 and 8 bits using NNCF `:math:`\Uparrow` <#table-of-content>`__ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -534,11 +535,11 @@ improves performance even more, but introduces a minor drop in prediction quality. More details about weights compression, can be found in `OpenVINO -documentation `__. +documentation `__. **Note**: There is no speedup for INT4 compressed models on dGPU. -Convert model to OpenVINO IR format +Convert model to OpenVINO IR format `:math:`\Uparrow` <#table-of-content>`__ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/notebooks/257-llava-multimodal-chatbot-with-output_files/index.html b/docs/notebooks/257-llava-multimodal-chatbot-with-output_files/index.html index 76276b096a4..bdda9c41894 100644 --- a/docs/notebooks/257-llava-multimodal-chatbot-with-output_files/index.html +++ b/docs/notebooks/257-llava-multimodal-chatbot-with-output_files/index.html @@ -1,7 +1,7 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/257-llava-multimodal-chatbot-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/257-llava-multimodal-chatbot-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/257-llava-multimodal-chatbot-with-output_files/


../
-257-llava-multimodal-chatbot-with-output_20_1.png  15-Nov-2023 00:43              539244
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/257-llava-multimodal-chatbot-with-output_files/


../
+257-llava-multimodal-chatbot-with-output_20_1.png  07-Dec-2023 00:49              539244
 

diff --git a/docs/notebooks/260-pix2struct-docvqa-with-output.rst b/docs/notebooks/260-pix2struct-docvqa-with-output.rst index 73926ec941d..8cb08a85db2 100644 --- a/docs/notebooks/260-pix2struct-docvqa-with-output.rst +++ b/docs/notebooks/260-pix2struct-docvqa-with-output.rst @@ -44,6 +44,7 @@ convert the model to OpenVINO™ IR format. **Table of contents:** + - `About Pix2Struct <#about-pixstruct>`__ - `Prerequisites <#prerequisites>`__ - `Download and Convert Model <#download-and-convert-model>`__ @@ -99,7 +100,7 @@ documentation `__. .. code:: ipython3 %pip install -q torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu - %pip install -q "git+https://github.com/huggingface/optimum-intel.git" "openvino>=2023.1.0" "transformers>=4.33.0" onnx gradio + %pip install -q "git+https://github.com/huggingface/optimum-intel.git" "openvino>=2023.1.0" "transformers>=4.33.0" onnx gradio --extra-index-url https://download.pytorch.org/whl/cpu Download and Convert Model -------------------------- @@ -230,7 +231,7 @@ by ``Pix2StructProcessor.decode`` Let’s see the model in action. For testing the model, we will use a screenshot from `OpenVINO -documentation `__ +documentation `__ .. code:: ipython3 diff --git a/docs/notebooks/260-pix2struct-docvqa-with-output_files/index.html b/docs/notebooks/260-pix2struct-docvqa-with-output_files/index.html index 1728691a60f..216ad429250 100644 --- a/docs/notebooks/260-pix2struct-docvqa-with-output_files/index.html +++ b/docs/notebooks/260-pix2struct-docvqa-with-output_files/index.html @@ -1,8 +1,8 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/260-pix2struct-docvqa-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/260-pix2struct-docvqa-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/260-pix2struct-docvqa-with-output_files/


../
-260-pix2struct-docvqa-with-output_11_0.jpg         15-Nov-2023 00:43              134092
-260-pix2struct-docvqa-with-output_11_0.png         15-Nov-2023 00:43              221889
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/260-pix2struct-docvqa-with-output_files/


../
+260-pix2struct-docvqa-with-output_11_0.jpg         07-Dec-2023 00:49              134092
+260-pix2struct-docvqa-with-output_11_0.png         07-Dec-2023 00:49              221889
 

diff --git a/docs/notebooks/261-fast-segment-anything-with-output.rst b/docs/notebooks/261-fast-segment-anything-with-output.rst index 4710119100e..21ca1341195 100644 --- a/docs/notebooks/261-fast-segment-anything-with-output.rst +++ b/docs/notebooks/261-fast-segment-anything-with-output.rst @@ -30,6 +30,7 @@ the prompt. **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Install requirements <#install-requirements>`__ @@ -65,7 +66,7 @@ Install requirements .. code:: ipython3 - %pip install -q "ultralytics==8.0.200" onnx + %pip install -q "ultralytics==8.0.200" onnx --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "openvino-dev>=2023.1.0" %pip install -q "nncf>=2.6.0" %pip install -q gradio @@ -156,8 +157,8 @@ model and generate a segmentation map. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/notebooks/261-fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 631.0ms - Speed: 3.8ms preprocess, 631.0ms inference, 21.8ms postprocess per image at shape (1, 3, 768, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/261-fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 621.3ms + Speed: 3.7ms preprocess, 621.3ms inference, 22.9ms postprocess per image at shape (1, 3, 768, 1024) The model returns segmentation maps for all the objects on the image. @@ -201,13 +202,13 @@ tracing. The FastSAM model itself is based on YOLOv8 model. PyTorch: starting from 'FastSAM-x.pt' with input shape (1, 3, 1024, 1024) BCHW and output shape(s) ((1, 37, 21504), (1, 32, 256, 256)) (138.2 MB) ONNX: starting export with onnx 1.15.0 opset 16... - ONNX: export success ✅ 3.5s, saved as 'FastSAM-x.onnx' (275.5 MB) + ONNX: export success ✅ 3.4s, saved as 'FastSAM-x.onnx' (275.5 MB) - OpenVINO: starting export with openvino 2023.1.0-12185-9e6b00e51cd-releases/2023/1... - OpenVINO: export success ✅ 1.0s, saved as 'FastSAM-x_openvino_model/' (275.9 MB) + OpenVINO: starting export with openvino 2023.2.0-13089-cfd42bd2cb0-HEAD... + OpenVINO: export success ✅ 1.1s, saved as 'FastSAM-x_openvino_model/' (275.9 MB) - Export complete (7.5s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/notebooks/261-fast-segment-anything + Export complete (7.4s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/261-fast-segment-anything Predict: yolo predict task=segment model=FastSAM-x_openvino_model imgsz=1024 Validate: yolo val task=segment model=FastSAM-x_openvino_model imgsz=1024 data=ultralytics/datasets/sa.yaml Visualize: https://netron.app @@ -306,8 +307,8 @@ pipeline. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/notebooks/261-fast-segment-anything/coco_bike.jpg: 480x640 33 objects, 353.6ms - Speed: 3.5ms preprocess, 353.6ms inference, 14.7ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/261-fast-segment-anything/coco_bike.jpg: 480x640 33 objects, 321.2ms + Speed: 2.3ms preprocess, 321.2ms inference, 20.1ms postprocess per image at shape (1, 3, 480, 640) One can observe the converted model outputs in the next cell, they is @@ -376,7 +377,7 @@ The quantization algorithm is based on `The YOLOv8 quantization example `__ in the NNCF repo, refer there for more details. Moreover, you can check out other quantization tutorials in the `OV notebooks -repo `__. +repo <../230-yolov8-optimization/>`__. **Note**: Model post-training quantization is time-consuming process. Be patient, it can take several minutes depending on your hardware. @@ -534,20 +535,62 @@ repo
+ + + + +.. raw:: html + +
+    
+ + + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:80: RuntimeWarning: invalid value encountered in multiply + return Tensor(self.data * unwrap_tensor_data(other)) + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ Compare the performance of the Original and Quantized Models diff --git a/docs/notebooks/261-fast-segment-anything-with-output_files/index.html b/docs/notebooks/261-fast-segment-anything-with-output_files/index.html index 5e233240faa..c6440fc6bd3 100644 --- a/docs/notebooks/261-fast-segment-anything-with-output_files/index.html +++ b/docs/notebooks/261-fast-segment-anything-with-output_files/index.html @@ -1,10 +1,10 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/261-fast-segment-anything-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/261-fast-segment-anything-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/261-fast-segment-anything-with-output_files/


../
-261-fast-segment-anything-with-output_21_0.jpg     15-Nov-2023 00:43              116049
-261-fast-segment-anything-with-output_21_0.png     15-Nov-2023 00:43              824318
-261-fast-segment-anything-with-output_9_0.jpg      15-Nov-2023 00:43              117489
-261-fast-segment-anything-with-output_9_0.png      15-Nov-2023 00:43              815077
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/261-fast-segment-anything-with-output_files/


../
+261-fast-segment-anything-with-output_21_0.jpg     07-Dec-2023 00:49              116049
+261-fast-segment-anything-with-output_21_0.png     07-Dec-2023 00:49              824318
+261-fast-segment-anything-with-output_9_0.jpg      07-Dec-2023 00:49              117489
+261-fast-segment-anything-with-output_9_0.png      07-Dec-2023 00:49              815077
 

diff --git a/docs/notebooks/262-softvc-voice-conversion-with-output.rst b/docs/notebooks/262-softvc-voice-conversion-with-output.rst index a333dba3adb..41946b1b944 100644 --- a/docs/notebooks/262-softvc-voice-conversion-with-output.rst +++ b/docs/notebooks/262-softvc-voice-conversion-with-output.rst @@ -18,8 +18,8 @@ audio are preserved. In this tutorial we will use the base model flow. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Use the original model to run an @@ -36,7 +36,7 @@ Prerequisites .. code:: ipython3 %pip install -q --upgrade pip setuptools - %pip install -q "openvino>=2023.2.0.dev20230922" + %pip install -q "openvino>=2023.2.0" !git clone https://github.com/svc-develop-team/so-vits-svc -b 4.1-Stable %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu tqdm librosa "torch>=2.1.0" "torchaudio>=2.1.0" faiss-cpu gradio "numpy==1.23.5" "fairseq==0.12.2" praat-parselmouth diff --git a/docs/notebooks/263-latent-consistency-models-image-generation-with-output.rst b/docs/notebooks/263-latent-consistency-models-image-generation-with-output.rst index 3caebe1dcd1..14293eda295 100644 --- a/docs/notebooks/263-latent-consistency-models-image-generation-with-output.rst +++ b/docs/notebooks/263-latent-consistency-models-image-generation-with-output.rst @@ -42,24 +42,35 @@ repository `__. In this tutorial, we consider how to convert and run LCM using OpenVINO. An additional part demonstrates how to run quantization with `NNCF `__ to speed up -pipeline. +pipeline. **Table of contents:** -- `Prerequisites <#prerequisites>`__ -- `Prepare models for OpenVINO format conversion <#prepare-models-for-openvino-format-conversion>`__ -- `Convert models to OpenVINO format <#convert-models-to-openvino-format>`__ -- `Text Encoder <#text-encoder>`__ -- `U-Net <#u-net>`__ -- `VAE <#vae>`__ -- `Prepare inference pipeline <#prepare-inference-pipeline>`__ -- `Configure Inference Pipeline <#configure-inference-pipeline>`__ -- `Text-to-image generation <#text-to-image-generation>`__ -- `Quantization <#quantization>`__ -- `Prepare calibration dataset <#prepare-calibration-dataset>`__ -- `Run quantization <#run-quantization>`__ -- `Compare inference time of the FP16 and INT8 models <#compare-inference-time-of-the-fp-and-int-models>`__ -- `Interactive demo <#interactive-demo>`__ + +- `Prerequisites <#prerequisites>`__ +- `Prepare models for OpenVINO format + conversion <#prepare-models-for-openvino-format-conversion>`__ +- `Convert models to OpenVINO + format <#convert-models-to-openvino-format>`__ + + - `Text Encoder <#text-encoder>`__ + - `U-Net <#u-net>`__ + - `VAE <#vae>`__ + +- `Prepare inference pipeline <#prepare-inference-pipeline>`__ + + - `Configure Inference Pipeline <#configure-inference-pipeline>`__ + +- `Text-to-image generation <#text-to-image-generation>`__ +- `Quantization <#quantization>`__ + + - `Prepare calibration dataset <#prepare-calibration-dataset>`__ + - `Run quantization <#run-quantization>`__ + - `Compare inference time of the FP16 and INT8 + models <#compare-inference-time-of-the-fp-and-int-models>`__ + - `Compare UNet file size <#compare-unet-file-size>`__ + +- `Interactive demo <#interactive-demo>`__ Prerequisites ------------- @@ -69,7 +80,7 @@ Prerequisites .. code:: ipython3 %pip install -q "torch" --index-url https://download.pytorch.org/whl/cpu - %pip install -q "openvino>=2023.1.0" transformers "diffusers>=0.22.0" pillow gradio "nncf>=2.6.0" datasets + %pip install -q "openvino>=2023.1.0" transformers "diffusers>=0.23.1" pillow gradio "nncf>=2.6.0" datasets --extra-index-url https://download.pytorch.org/whl/cpu Prepare models for OpenVINO format conversion --------------------------------------------- @@ -88,7 +99,7 @@ model is also integrated into Diffusers is the go-to library for state-of-the-art pretrained diffusion models for generating images, audio, and even 3D structures of molecules. This allows us to compare running original Stable Diffusion -(from this `notebook <225-stable-diffusion-text-to-image-with-output.html>`__) and +(from this `notebook <../225-stable-diffusion-text-to-image>`__) and distilled using LCD. The distillation approach efficiently converts a pre-trained guided diffusion model into a latent consistency model by solving an augmented PF-ODE. @@ -116,7 +127,7 @@ provide which module should be loaded for initialization using VAE_DECODER_OV_PATH = Path("model/vae_decoder.xml") - def load_orginal_pytorch_pipeline_componets(skip_models=False, skip_safety_checker=True): + def load_orginal_pytorch_pipeline_componets(skip_models=False, skip_safety_checker=False): pipe = DiffusionPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7") scheduler = pipe.scheduler tokenizer = pipe.tokenizer @@ -167,11 +178,6 @@ provide which module should be loaded for initialization using Loading pipeline components...: 0%| | 0/7 [00:00 x_t-1 latents, denoised = self.scheduler.step( @@ -735,7 +699,7 @@ inference using OpenVINO. .. parsed-literal:: - Dropdown(description='Device:', options=('CPU', 'GPU', 'AUTO'), value='CPU') + Dropdown(description='Device:', options=('CPU', 'AUTO'), value='CPU') @@ -890,9 +854,11 @@ model inputs for calibration we should customize ``CompiledModel``. import datasets from tqdm.notebook import tqdm - from transformers import Pipeline + from transformers import set_seed from typing import Any, Dict, List + set_seed(1) + class CompiledModelDecorator(ov.CompiledModel): def __init__(self, compiled_model, prob: float, data_cache: List[Any] = None): super().__init__(compiled_model) @@ -904,18 +870,22 @@ model inputs for calibration we should customize ``CompiledModel``. self.data_cache.append(*args) return super().__call__(*args, **kwargs) - def collect_calibration_data(lcm_pipeline: Pipeline, subset_size: int) -> List[Dict]: + def collect_calibration_data(lcm_pipeline: OVLatentConsistencyModelPipeline, subset_size: int) -> List[Dict]: original_unet = lcm_pipeline.unet lcm_pipeline.unet = CompiledModelDecorator(original_unet, prob=0.3) dataset = datasets.load_dataset("laion/laion2B-en", split="train", streaming=True).shuffle(seed=42) lcm_pipeline.set_progress_bar_config(disable=True) + safety_checker = lcm_pipeline.safety_checker + lcm_pipeline.safety_checker = None # Run inference for data collection pbar = tqdm(total=subset_size) diff = 0 for batch in dataset: prompt = batch["TEXT"] + if len(prompt) > tokenizer.model_max_length: + continue _ = lcm_pipeline( prompt, num_inference_steps=num_inference_steps, @@ -935,6 +905,7 @@ model inputs for calibration we should customize ``CompiledModel``. calibration_dataset = lcm_pipeline.unet.data_cache lcm_pipeline.set_progress_bar_config(disable=False) lcm_pipeline.unet = original_unet + lcm_pipeline.safety_checker = safety_checker return calibration_dataset .. code:: ipython3 @@ -954,7 +925,7 @@ model inputs for calibration we should customize ``CompiledModel``. .. parsed-literal:: - Downloading readme: 0%| | 0.00/56.0 [00:00 77). Running this sequence through the model will result in indexing errors - WARNING:__main__:The following part of your input was truncated because CLIP can only handle sequences up to 77 tokens: ['colleges harnessing technology to make education free'] - - Run quantization ~~~~~~~~~~~~~~~~ @@ -1004,25 +969,72 @@ Create a quantized model from the pre-trained converted OpenVINO model. ov.save_model(quantized_unet, UNET_INT8_OV_PATH) + .. parsed-literal:: - INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + .. parsed-literal:: - Statistics collection: 100%|██████████| 200/200 [03:15<00:00, 1.02it/s] - Applying Smooth Quant: 100%|██████████| 101/101 [00:07<00:00, 13.89it/s] + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ .. parsed-literal:: - INFO:nncf:96 ignored nodes was found by name in the NNCFGraph + INFO:nncf:96 ignored nodes were found by name in the NNCFGraph + .. parsed-literal:: - Statistics collection: 100%|██████████| 200/200 [03:57<00:00, 1.19s/it] + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ .. code:: ipython3 @@ -1098,8 +1110,7 @@ pipelines, we use median inference time on calibration subset. validation_size = 10 calibration_dataset = datasets.load_dataset("laion/laion2B-en", split="train", streaming=True).take(validation_size) validation_data = [] - while len(validation_data) < validation_size: - batch = next(iter(calibration_dataset)) + for batch in calibration_dataset: prompt = batch["TEXT"] validation_data.append(prompt) @@ -1122,6 +1133,13 @@ pipelines, we use median inference time on calibration subset. inference_time.append(delta) return np.median(inference_time) + + +.. parsed-literal:: + + Resolving data files: 0%| | 0/128 [00:00 -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/263-latent-consistency-models-image-generation-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/263-latent-consistency-models-image-generation-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/263-latent-consistency-models-image-generation-with-output_files/


../
-263-latent-consistency-models-image-generation-..> 15-Nov-2023 00:43              345052
-263-latent-consistency-models-image-generation-..> 15-Nov-2023 00:43              352748
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/263-latent-consistency-models-image-generation-with-output_files/


../
+263-latent-consistency-models-image-generation-..> 07-Dec-2023 00:49               20243
+263-latent-consistency-models-image-generation-..> 07-Dec-2023 00:49              390255
+263-latent-consistency-models-image-generation-..> 07-Dec-2023 00:49               20298
+263-latent-consistency-models-image-generation-..> 07-Dec-2023 00:49              393756
 

diff --git a/docs/notebooks/263-lcm-lora-controlnet-with-output.rst b/docs/notebooks/263-lcm-lora-controlnet-with-output.rst new file mode 100644 index 00000000000..39fb259bb01 --- /dev/null +++ b/docs/notebooks/263-lcm-lora-controlnet-with-output.rst @@ -0,0 +1,1664 @@ +Text-to-Image Generation with LCM LoRA and ControlNet Conditioning +================================================================== + +Diffusion models make a revolution in AI-generated art. This technology +enables the creation of high-quality images simply by writing a text +prompt. Even though this technology gives very promising results, the +diffusion process, in the first order, is the process of generating +images from random noise and text conditions, which do not always +clarify how desired content should look, which forms it should have, and +where it is located in relation to other objects on the image. +Researchers have been looking for ways to have more control over the +results of the generation process. ControlNet provides a minimal +interface allowing users to customize the generation process to a great +extent. + +ControlNet was introduced in `Adding Conditional Control to +Text-to-Image Diffusion Models `__ +paper. It provides a framework that enables support for various spatial +contexts such as a depth map, a segmentation map, a scribble, and key +points that can serve as additional conditionings to Diffusion models +such as Stable Diffusion. + +Latent Consistency Models (LCM) are a way to decrease the number of +steps required to generate an image with Stable Diffusion (or SDXL) by +distilling the original model into another version that requires fewer +steps (4 to 8 instead of the original 25 to 50). Distillation is a type +of training procedure that attempts to replicate the outputs from a +source model using a new one. The distilled model may be designed to be +smaller or, in this case, require fewer steps to run. It’s usually a +lengthy and costly process that requires huge amounts of data, patience, +and powerful training hardware. + +For latent consistency distillation, each model needs to be distilled +separately. The LCM LoRA allows to train just a small number of +adapters, known as LoRA layers, instead of the full model. The resulting +LoRAs can then be applied to any fine-tuned version of the model without +having to distil them separately. The benefit of this LCM LoRA +distillation process is that it can be integrated into the existing +inference pipelines without changes to the main code, for example, into +the ControlNet-guided Stable Diffusion pipeline. More details about LCM +LoRA can be found in the `technical +report `__ and `blog +post `__ + +This notebook explores how to speed up ControlNet pipeline using LCM +LoRA, OpenVINO and quantization with +`NNCF `__. Let us get +“controlling”! + + +**Table of contents:** + +- `Background <#background>`__ +- `Stable Diffusion <#stable-diffusion>`__ +- `controlnet <#controlnet>`__ +- `Low-Rank Adaptation of Large Language Models (LoRA) <#low-rank-adaptation-of-large-language-models-lora>`__ +- `Prerequisites <#prerequisites>`__ +- `Load Original Diffusers pipeline and prepare models for conversion <#load-original-diffusers-pipeline-and-prepare-models-for-conversion>`__ +- `Condition Image <#condition-image>`__ +- `Convert models to OpenVINO Intermediate representation (IR) format <#convert-models-to-openvino-intermediate-representation-ir-format>`__ +- `ControlNet conversion <#controlnet-conversion>`__ +- `U-Net <#u-net>`__ +- `textencoder <#text-encoder>`__ +- `VAE Decoder conversion <#vae-decoder-conversion>`__ +- `Prepare Inference pipeline <#prepare-inference-pipeline>`__ +- `Prepare tokenizer and LCMScheduler <#prepare-tokenizer-and-lcmscheduler>`__ +- `Select inference device for Stable Diffusion pipeline <#select-inference-device-for-stable-diffusion-pipeline>`__ +- `Running Text-to-Image Generation with ControlNet Conditioning and OpenVINO <#running-text-to-image-generation-with-controlnet-conditioning-and-openvino>`__ +- `Quantization <#quantization>`__ +- `Prepare calibration datasets <#prepare-calibration-datasets>`__ +- `Run quantization <#run-quantization>`__ +- `Compare inference time of the FP16 and INT8 models <#compare-inference-time-of-the-fp-and-int-models>`__ +- `Compare model file sizes <#compare-model-file-sizes>`__ +- `Interactive Demo <#interactive-demo>`__ + +Background +---------- + + + +Stable Diffusion +~~~~~~~~~~~~~~~~ + + + +`Stable Diffusion `__ is a +text-to-image latent diffusion model created by researchers and +engineers from CompVis, Stability AI, and LAION. Diffusion models as +mentioned above can generate high-quality images. Stable Diffusion is +based on a particular type of diffusion model called Latent Diffusion, +proposed in `High-Resolution Image Synthesis with Latent Diffusion +Models `__ paper. Generally speaking, +diffusion models are machine learning systems that are trained to +denoise random Gaussian noise step by step, to get to a sample of +interest, such as an image. Diffusion models have been shown to achieve +state-of-the-art results for generating image data. But one downside of +diffusion models is that the reverse denoising process is slow because +of its repeated, sequential nature. In addition, these models consume a +lot of memory because they operate in pixel space, which becomes huge +when generating high-resolution images. Latent diffusion can reduce the +memory and compute complexity by applying the diffusion process over a +lower dimensional latent space, instead of using the actual pixel space. +This is the key difference between standard diffusion and latent +diffusion models: in latent diffusion, the model is trained to generate +latent (compressed) representations of the images. + +There are three main components in latent diffusion: + +- A text-encoder, for example `CLIP’s Text + Encoder `__ + for creation condition to generate image from text prompt. +- A U-Net for step-by-step denoising latent image representation. +- An autoencoder (VAE) for encoding input image to latent space (if + required) and decoding latent space to image back after generation. + +For more details regarding Stable Diffusion work, refer to the `project +website `__. +There is a tutorial for Stable Diffusion Text-to-Image generation with +OpenVINO, see the following +`notebook <225-stable-diffusion-text-to-image-with-output.html>`__. + +ControlNet +~~~~~~~~~~ + +ControlNet is a neural network +structure to control diffusion models by adding extra conditions. Using +this new framework, we can capture a scene, structure, object, or +subject pose from an inputted image, and then transfer that quality to +the generation process. In practice, this enables the model to +completely retain the original input shape, and create a novel image +that conserves the shape, pose, or outline while using the novel +features from the inputted prompt. + +.. figure:: https://raw.githubusercontent.com/lllyasviel/ControlNet/main/github_page/he.png + :alt: controlnet block + + controlnet block + +Functionally, ControlNet operates by wrapping around an image synthesis +process to impart attention to the shape required to operate the model +using either its inbuilt prediction or one of many additional annotator +models. Referring to the diagram above, we can see, on a rudimentary +level, how ControlNet uses a trainable copy in conjunction with the +original network to modify the final output with respect to the shape of +the input control source. + +By repeating the above simple structure 14 times, we can control stable +diffusion in the following way: + +.. figure:: https://raw.githubusercontent.com/lllyasviel/ControlNet/main/github_page/sd.png + :alt: sd + controlnet + + sd + controlnet + +The input is simultaneously passed through the SD blocks, represented on +the left, while simultaneously being processed by the ControlNet blocks +on the right. This process is almost the same during encoding. When +denoising the image, at each step the SD decoder blocks will receive +control adjustments from the parallel processing path from ControlNet. + +In the end, we are left with a very similar image synthesis pipeline +with an additional control added for the shape of the output features in +the final image. + +Low-Rank Adaptation of Large Language Models (LoRA) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +`Low-Rank Adaptation of Large Language Models +(LoRA) `__ is a training method that +accelerates the training of large models while consuming less memory. It +adds pairs of rank-decomposition weight matrices (called update +matrices) to existing weights, and only trains those newly added +weights. This has a couple of advantages: + +- LoRA makes fine-tuning more efficient by drastically reducing the + number of trainable parameters. +- The original pre-trained weights are kept frozen, which means you can + have multiple lightweight and portable LoRA models for various + downstream tasks built on top of them. +- LoRA is orthogonal to many other parameter-efficient methods and can + be combined with many of them. +- Performance of models fine-tuned using LoRA is comparable to the + performance of fully fine-tuned models. +- LoRA does not add any inference latency because adapter weights can + be merged with the base model. + +In principle, LoRA can be applied to any subset of weight matrices in a +neural network to reduce the number of trainable parameters. However, +for simplicity and further parameter efficiency, in Transformer models +LoRA is typically applied to attention blocks only. The resulting number +of trainable parameters in a LoRA model depends on the size of the +low-rank update matrices, which is determined mainly by the rank r and +the shape of the original weight matrix. More details about LoRA can be +found in HuggingFace `conceptual +guide `__, +`Diffusers +documentation `__ +and `blog post `__. + +Prerequisites +------------- + + + +Install required packages + +.. code:: ipython3 + + %pip install -q "torch" transformers "diffusers>=0.22.0" "controlnet-aux>=0.0.6" accelerate --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "openvino>=2023.2.0" pillow gradio datasets "nncf>=2.7.0" + +Prepare PyTorch models + +.. code:: ipython3 + + from pathlib import Path + + controlnet_id = "lllyasviel/control_v11p_sd15_normalbae" + adapter_id = "latent-consistency/lcm-lora-sdv1-5" + stable_diffusion_id = "runwayml/stable-diffusion-v1-5" + + TEXT_ENCODER_OV_PATH = Path('model/text_encoder.xml') + UNET_OV_PATH = Path('model/unet_controlnet.xml') + CONTROLNET_OV_PATH = Path('model/controlnet-normalbae.xml') + VAE_DECODER_OV_PATH = Path('model/vae_decoder.xml') + TOKENIZER_PATH = Path('model/tokenizer') + SCHEDULER_PATH = Path('model/scheduler') + + skip_models = TEXT_ENCODER_OV_PATH.exists() and UNET_OV_PATH.exists() and CONTROLNET_OV_PATH.exists() and VAE_DECODER_OV_PATH.exists() + +Load Original Diffusers pipeline and prepare models for conversion +------------------------------------------------------------------ + + + +For working with Stable Diffusion and ControlNet models, we will use +Hugging Face `Diffusers `__ +library. To experiment with ControlNet, Diffusers exposes the +`StableDiffusionControlNetPipeline `__ +similar to the `other Diffusers +pipelines `__. +Central to the ``StableDiffusionControlNetPipeline`` is the +``controlnet`` argument which enables providing a particularly trained +`ControlNetModel `__ +instance while keeping the pre-trained diffusion model weights the same. + +The code below demonstrates how to create +``StableDiffusionControlNetPipeline``. The process consists of the +following steps: 1. Create ``ControlNetModel`` for passing to pipeline +using ``from_pretrained`` method. 2. Create +``StableDiffusionControlNetPipeline`` using Stable Diffusion and +ControlNet model 3. Load LoRA weights to the pipeline using +``load_lora_weights`` method. + +.. code:: ipython3 + + from diffusers import StableDiffusionControlNetPipeline, ControlNetModel + import gc + + + def load_original_pytorch_pipeline_components(controlnet_id:str, stable_diffusion_id:str, adapter_id:str): + """ + Helper function for loading Stable Diffusion ControlNet pipeline and applying LCM LoRA + + Parameters: + controlnet_id: model id from HuggingFace hub or local path for loading ControlNet model + stable_diffusion_id: model id from HuggingFace hub or local path for loading Stable Diffusion model + adapter_id: LCM LoRA id from HuggingFace hub or local path + Returns: + controlnet: ControlNet model + text_encoder: Stable Diffusion Text Encoder + unet: Stable Diffusion U-Net + vae: Stable Diffusion Variational Autoencoder (VAE) + """ + + # load controlnet model + controlnet = ControlNetModel.from_pretrained(controlnet_id) + # load stable diffusion pipeline + pipe = StableDiffusionControlNetPipeline.from_pretrained(stable_diffusion_id, controlnet=controlnet) + # load LCM LoRA weights + pipe.load_lora_weights(adapter_id) + # fuse LoRA weights with UNet + pipe.fuse_lora() + text_encoder = pipe.text_encoder + text_encoder.eval() + unet = pipe.unet + unet.eval() + vae = pipe.vae + vae.eval() + del pipe + gc.collect() + return controlnet, text_encoder, unet, vae + +.. code:: ipython3 + + controlnet, text_encoder, unet, vae = None, None, None, None + if not skip_models: + controlnet, text_encoder, unet, vae = load_original_pytorch_pipeline_components(controlnet_id, stable_diffusion_id, adapter_id) + +Condition Image +--------------- + + + +The process of extracting specific information from the input image is +called an annotation. ControlNet comes pre-packaged with compatibility +with several annotators-models that help it to identify the shape/form +of the target in the image: + +- Canny Edge Detection +- M-LSD Lines +- HED Boundary +- Scribbles +- Normal Map +- Human Pose Estimation +- Semantic Segmentation +- Depth Estimation + +In this tutorial we will use `Normal +Mapping `__ for +controlling diffusion process. For this case, ControlNet condition image +is an image with surface normal information, usually represented as a +color-coded image. + +.. code:: ipython3 + + from controlnet_aux import NormalBaeDetector + from diffusers.utils import load_image + from urllib.request import urlretrieve + import matplotlib.pyplot as plt + from PIL import Image + import numpy as np + + example_image_url = "https://huggingface.co/lllyasviel/control_v11p_sd15_normalbae/resolve/main/images/input.png" + urlretrieve(example_image_url, "example.png") + + processor = NormalBaeDetector.from_pretrained("lllyasviel/Annotators") + + image = load_image("example.png") + control_image = processor(image) + + + def visualize_results(orig_img:Image.Image, normal_img:Image.Image, result_img:Image.Image = None, save_fig:bool = False): + """ + Helper function for results visualization + + Parameters: + orig_img (Image.Image): original image + normal_img (Image.Image): image with bwith surface normal information + result_img (Image.Image, optional, default None): generated image + safe_fig (bool, optional, default False): allow saving visualization result on disk + Returns: + fig (matplotlib.pyplot.Figure): matplotlib generated figure contains drawing result + """ + orig_title = "Original image" + control_title = "Normal map" + orig_img = orig_img.resize(normal_img.size if result_img is None else result_img.size) + im_w, im_h = orig_img.size + is_horizontal = im_h <= im_w + figsize = (20, 20) + num_images = 3 if result_img is not None else 2 + fig, axs = plt.subplots(num_images if is_horizontal else 1, 1 if is_horizontal else num_images, figsize=figsize, sharex='all', sharey='all') + fig.patch.set_facecolor('white') + list_axes = list(axs.flat) + for a in list_axes: + a.set_xticklabels([]) + a.set_yticklabels([]) + a.get_xaxis().set_visible(False) + a.get_yaxis().set_visible(False) + a.grid(False) + list_axes[0].imshow(np.array(orig_img)) + list_axes[1].imshow(np.array(normal_img)) + list_axes[0].set_title(orig_title, fontsize=15) + list_axes[1].set_title(control_title, fontsize=15) + if result_img is not None: + list_axes[2].imshow(np.array(result_img)) + list_axes[2].set_title("Result", fontsize=15) + + fig.subplots_adjust(wspace=0.01 if is_horizontal else 0.00 , hspace=0.01 if is_horizontal else 0.1) + fig.tight_layout() + if save_fig: + fig.savefig("result.png", bbox_inches='tight') + return fig + + + fig = visualize_results(image, control_image) + + +.. parsed-literal:: + + Loading base model ()...Done. + Removing last two layers (global_pool & classifier). + + + +.. image:: 263-lcm-lora-controlnet-with-output_files/263-lcm-lora-controlnet-with-output_10_1.png + + +Convert models to OpenVINO Intermediate representation (IR) format +------------------------------------------------------------------ + + + +Starting from 2023.0 release, OpenVINO supports PyTorch models +conversion directly. We need to provide a model object, input data for +model tracing to ``ov.convert_model`` function to obtain OpenVINO +``ov.Model`` object instance. Model can be saved on disk for next +deployment using ``ov.save_model`` function. + +The pipeline consists of five important parts: + +- ControlNet for conditioning by image annotation. +- Text Encoder for creation condition to generate an image from a text + prompt. +- Unet for step-by-step denoising latent image representation. +- Autoencoder (VAE) for decoding latent space to image. + +Let us convert each part: + +ControlNet conversion +~~~~~~~~~~~~~~~~~~~~~ + + + +The ControlNet model accepts the same inputs like UNet in Stable +Diffusion pipeline and additional condition sample - skeleton key points +map predicted by pose estimator: + +- ``sample`` - latent image sample from the previous step, generation + process has not been started yet, so we will use random noise, +- ``timestep`` - current scheduler step, +- ``encoder_hidden_state`` - hidden state of text encoder, +- ``controlnet_cond`` - condition input annotation. + +The output of the model is attention hidden states from down and middle +blocks, which serves additional context for the UNet model. + +.. code:: ipython3 + + import torch + import openvino as ov + from functools import partial + + def cleanup_torchscript_cache(): + """ + Helper for removing cached model representation + """ + torch._C._jit_clear_class_registry() + torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() + torch.jit._state._clear_class_state() + + + def flattenize_inputs(inputs): + """ + Helper function for resolve nested input structure (e.g. lists or tuples of tensors) + """ + flatten_inputs = [] + for input_data in inputs: + if input_data is None: + continue + if isinstance(input_data, (list, tuple)): + flatten_inputs.extend(flattenize_inputs(input_data)) + else: + flatten_inputs.append(input_data) + return flatten_inputs + + + dtype_mapping = { + torch.float32: ov.Type.f32, + torch.float64: ov.Type.f64, + torch.int32: ov.Type.i32, + torch.int64: ov.Type.i64 + } + + + def prepare_input_info(input_dict): + """ + Helper function for preparing input info (shapes and data types) for conversion based on example inputs + """ + flatten_inputs = flattenize_inputs(inputs.values()) + input_info = [] + for input_data in flatten_inputs: + updated_shape = list(input_data.shape) + if updated_shape: + updated_shape[0] = -1 + if input_data.ndim == 4: + updated_shape[2] = -1 + updated_shape[3] = -1 + + input_info.append((dtype_mapping[input_data.dtype], updated_shape)) + return input_info + + + inputs = { + "sample": torch.randn((1, 4, 64, 64)), + "timestep": torch.tensor(1, dtype=torch.float32), + "encoder_hidden_states": torch.randn((1,77,768)), + "controlnet_cond": torch.randn((1,3,512,512)) + } + + + # Prepare conditional inputs for U-Net + if not UNET_OV_PATH.exists(): + controlnet.eval() + with torch.no_grad(): + down_block_res_samples, mid_block_res_sample = controlnet(**inputs, return_dict=False) + + if not CONTROLNET_OV_PATH.exists(): + input_info = prepare_input_info(inputs) + with torch.no_grad(): + controlnet.forward = partial(controlnet.forward, return_dict=False) + ov_model = ov.convert_model(controlnet, example_input=inputs, input=input_info) + ov.save_model(ov_model, CONTROLNET_OV_PATH) + del ov_model + cleanup_torchscript_cache() + print('ControlNet successfully converted to IR') + else: + print(f"ControlNet will be loaded from {CONTROLNET_OV_PATH}") + + del controlnet + gc.collect() + + +.. parsed-literal:: + + ControlNet will be loaded from model/controlnet-normalbae.xml + + + + +.. parsed-literal:: + + 9 + + + +U-Net +~~~~~ + + + +The process of U-Net model conversion remains the same, like for +original Stable Diffusion model, but with respect to the new inputs +generated by ControlNet. + +.. code:: ipython3 + + from typing import Tuple + + class UnetWrapper(torch.nn.Module): + def __init__( + self, + unet, + sample_dtype=torch.float32, + timestep_dtype=torch.int64, + encoder_hidden_states=torch.float32, + down_block_additional_residuals=torch.float32, + mid_block_additional_residual=torch.float32 + ): + super().__init__() + self.unet = unet + self.sample_dtype = sample_dtype + self.timestep_dtype = timestep_dtype + self.encoder_hidden_states_dtype = encoder_hidden_states + self.down_block_additional_residuals_dtype = down_block_additional_residuals + self.mid_block_additional_residual_dtype = mid_block_additional_residual + + def forward( + self, + sample:torch.Tensor, + timestep:torch.Tensor, + encoder_hidden_states:torch.Tensor, + down_block_additional_residuals:Tuple[torch.Tensor], + mid_block_additional_residual:torch.Tensor + ): + sample.to(self.sample_dtype) + timestep.to(self.timestep_dtype) + encoder_hidden_states.to(self.encoder_hidden_states_dtype) + down_block_additional_residuals = [res.to(self.down_block_additional_residuals_dtype) for res in down_block_additional_residuals] + mid_block_additional_residual.to(self.mid_block_additional_residual_dtype) + return self.unet( + sample, + timestep, + encoder_hidden_states, + down_block_additional_residuals=down_block_additional_residuals, + mid_block_additional_residual=mid_block_additional_residual + ) + + + + if not UNET_OV_PATH.exists(): + inputs.pop("controlnet_cond", None) + inputs["down_block_additional_residuals"] = down_block_res_samples + inputs["mid_block_additional_residual"] = mid_block_res_sample + input_info = prepare_input_info(inputs) + + wrapped_unet = UnetWrapper(unet) + wrapped_unet.eval() + + with torch.no_grad(): + ov_model = ov.convert_model(wrapped_unet, example_input=inputs) + + for (input_dtype, input_shape), input_tensor in zip(input_info, ov_model.inputs): + input_tensor.get_node().set_partial_shape(ov.PartialShape(input_shape)) + input_tensor.get_node().set_element_type(input_dtype) + ov_model.validate_nodes_and_infer_types() + ov.save_model(ov_model, UNET_OV_PATH) + del ov_model + cleanup_torchscript_cache() + del wrapped_unet + del unet + gc.collect() + print('Unet successfully converted to IR') + else: + del unet + print(f"Unet will be loaded from {UNET_OV_PATH}") + gc.collect() + + +.. parsed-literal:: + + Unet will be loaded from model/unet_controlnet.xml + + + + +.. parsed-literal:: + + 0 + + + +Text Encoder +~~~~~~~~~~~~ + + + +The text-encoder is responsible for transforming the input prompt, for +example, “a photo of an astronaut riding a horse” into an embedding +space that can be understood by the U-Net. It is usually a simple +transformer-based encoder that maps a sequence of input tokens to a +sequence of latent text embeddings. + +The input of the text encoder is tensor ``input_ids``, which contains +indexes of tokens from text processed by the tokenizer and padded to the +maximum length accepted by the model. Model outputs are two tensors: +``last_hidden_state`` - hidden state from the last MultiHeadAttention +layer in the model and ``pooler_out`` - pooled output for whole model +hidden states. + +.. code:: ipython3 + + def convert_encoder(text_encoder:torch.nn.Module, ir_path:Path): + """ + Convert Text Encoder model to OpenVINO IR. + Function accepts text encoder model, prepares example inputs for conversion, and convert it to OpenVINO Model + Parameters: + text_encoder (torch.nn.Module): text_encoder model + ir_path (Path): File for storing model + Returns: + None + """ + if not ir_path.exists(): + input_ids = torch.ones((1, 77), dtype=torch.long) + # switch model to inference mode + text_encoder.eval() + + # disable gradients calculation for reducing memory consumption + with torch.no_grad(): + ov_model = ov.convert_model( + text_encoder, # model instance + example_input=input_ids, # inputs for model tracing + input=([1,77],) + ) + ov.save_model(ov_model, ir_path) + del ov_model + cleanup_torchscript_cache() + print('Text Encoder successfully converted to IR') + + + if not TEXT_ENCODER_OV_PATH.exists(): + convert_encoder(text_encoder, TEXT_ENCODER_OV_PATH) + else: + print(f"Text encoder will be loaded from {TEXT_ENCODER_OV_PATH}") + del text_encoder + gc.collect() + + +.. parsed-literal:: + + Text encoder will be loaded from model/text_encoder.xml + + + + +.. parsed-literal:: + + 0 + + + +VAE Decoder conversion +~~~~~~~~~~~~~~~~~~~~~~ + + + +The VAE model has two parts, an encoder, and a decoder. The encoder is +used to convert the image into a low-dimensional latent representation, +which will serve as the input to the U-Net model. The decoder, +conversely, transforms the latent representation back into an image. + +During latent diffusion training, the encoder is used to get the latent +representations (latents) of the images for the forward diffusion +process, which applies more and more noise at each step. During +inference, the denoised latents generated by the reverse diffusion +process are converted back into images using the VAE decoder. During +inference, we will see that we **only need the VAE decoder**. You can +find instructions on how to convert the encoder part in a stable +diffusion +`notebook <225-stable-diffusion-text-to-image-with-output.html>`__. + +.. code:: ipython3 + + def convert_vae_decoder(vae: torch.nn.Module, ir_path: Path): + """ + Convert VAE model to IR format. + Function accepts pipeline, creates wrapper class for export only necessary for inference part, + prepares example inputs for convert, + Parameters: + vae (torch.nn.Module): VAE model + ir_path (Path): File for storing model + Returns: + None + """ + class VAEDecoderWrapper(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, latents): + return self.vae.decode(latents) + + if not ir_path.exists(): + vae_decoder = VAEDecoderWrapper(vae) + latents = torch.zeros((1, 4, 64, 64)) + + vae_decoder.eval() + with torch.no_grad(): + ov_model = ov.convert_model(vae_decoder, example_input=latents, input=[-1, 4, -1, -1]) + ov.save_model(ov_model, ir_path) + del ov_model + cleanup_torchscript_cache() + print('VAE decoder successfully converted to IR') + + + if not VAE_DECODER_OV_PATH.exists(): + convert_vae_decoder(vae, VAE_DECODER_OV_PATH) + else: + print(f"VAE decoder will be loaded from {VAE_DECODER_OV_PATH}") + + del vae + + +.. parsed-literal:: + + VAE decoder will be loaded from model/vae_decoder.xml + + +Prepare Inference pipeline +-------------------------- + + + +We already deeply discussed how the ControlNet-guided pipeline works on +example pose-controlled generation in `controlnet +notebook <../235-controlnet-stable-diffusion>`__. In our current +example, the pipeline remains without changes. Similarly to Diffusers +``StableDiffusionControlNetPipeline``, we define our own +``OVControlNetStableDiffusionPipeline`` inference pipeline based on +OpenVINO. + +.. code:: ipython3 + + from diffusers import DiffusionPipeline + from transformers import CLIPTokenizer + from typing import Union, List, Optional, Tuple + import cv2 + + + def scale_fit_to_window(dst_width:int, dst_height:int, image_width:int, image_height:int): + """ + Preprocessing helper function for calculating image size for resize with peserving original aspect ratio + and fitting image to specific window size + + Parameters: + dst_width (int): destination window width + dst_height (int): destination window height + image_width (int): source image width + image_height (int): source image height + Returns: + result_width (int): calculated width for resize + result_height (int): calculated height for resize + """ + im_scale = min(dst_height / image_height, dst_width / image_width) + return int(im_scale * image_width), int(im_scale * image_height) + + + def preprocess(image: Image.Image, dst_height:int = 512, dst_width:int = 512): + """ + Image preprocessing function. Takes image in PIL.Image format, resizes it to keep aspect ration and fits to model input window 512x512, + then converts it to np.ndarray and adds padding with zeros on right or bottom side of image (depends from aspect ratio), after that + converts data to float32 data type and change range of values from [0, 255] to [-1, 1], finally, converts data layout from planar NHWC to NCHW. + The function returns preprocessed input tensor and padding size, which can be used in postprocessing. + + Parameters: + image (Image.Image): input image + dst_width: destination image width + dst_height: destination image height + Returns: + image (np.ndarray): preprocessed image tensor + pad (Tuple[int]): pading size for each dimension for restoring image size in postprocessing + """ + src_width, src_height = image.size + res_width, res_height = scale_fit_to_window(dst_width, dst_height, src_width, src_height) + image = np.array(image.resize((res_width, res_height), resample=Image.Resampling.LANCZOS))[None, :] + pad_width = dst_width - res_width + pad_height = dst_height - res_height + pad = ((0, 0), (0, pad_height), (0, pad_width), (0, 0)) + image = np.pad(image, pad, mode="constant") + image = image.astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + return image, pad + + + def randn_tensor( + shape: Union[Tuple, List], + dtype: Optional[torch.dtype] = torch.float32, + ): + """ + Helper function for generation random values tensor with given shape and data type + + Parameters: + shape (Union[Tuple, List]): shape for filling random values + dtype (torch.dtype, *optiona*, torch.float32): data type for result + Returns: + latents (np.ndarray): tensor with random values with given data type and shape (usually represents noise in latent space) + """ + latents = torch.randn(shape, dtype=dtype) + return latents.numpy() + + + class OVControlNetStableDiffusionPipeline(DiffusionPipeline): + """ + OpenVINO inference pipeline for Stable Diffusion with ControlNet guidence + """ + def __init__( + self, + tokenizer: CLIPTokenizer, + scheduler, + core: ov.Core, + controlnet: ov.Model, + text_encoder: ov.Model, + unet: ov.Model, + vae_decoder: ov.Model, + device:str = "AUTO" + ): + super().__init__() + self.tokenizer = tokenizer + self.vae_scale_factor = 8 + self.scheduler = scheduler + self.load_models(core, device, controlnet, text_encoder, unet, vae_decoder) + + def load_models(self, core: ov.Core, device: str, controlnet:ov.Model, text_encoder: ov.Model, unet: ov.Model, vae_decoder: ov.Model): + """ + Function for loading models on device using OpenVINO + + Parameters: + core (Core): OpenVINO runtime Core class instance + device (str): inference device + controlnet (Model): OpenVINO Model object represents ControlNet + text_encoder (Model): OpenVINO Model object represents text encoder + unet (Model): OpenVINO Model object represents UNet + vae_decoder (Model): OpenVINO Model object represents vae decoder + Returns + None + """ + self.text_encoder = core.compile_model(text_encoder, device) + self.register_to_config(controlnet=core.compile_model(controlnet, device)) + self.register_to_config(unet=core.compile_model(unet, device)) + ov_config = {"INFERENCE_PRECISION_HINT": "f32"} if device != "CPU" else {} + self.vae_decoder = core.compile_model(vae_decoder, device, ov_config) + + def __call__( + self, + prompt: Union[str, List[str]], + image: Image.Image, + num_inference_steps: int = 4, + height:int = 512, + width:int = 512, + negative_prompt: Union[str, List[str]] = None, + guidance_scale: float = 0.5, + controlnet_conditioning_scale: float = 1.0, + latents: Optional[np.array] = None, + output_type: Optional[str] = "pil", + ): + """ + Function invoked when calling the pipeline for generation. + + Parameters: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`Image.Image`): + `Image`, or tensor representing an image batch which will be repainted according to `prompt`. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (int, *optional*, defaults to 512): generated image height + width (int, *optional*, defaults to 512): generated image width + negative_prompt (`str` or `List[str]`): + negative prompt or prompts for generation + guidance_scale (`float`, *optional*, defaults to 0.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. This pipeline requires a value of at least `1`. + latents (`np.ndarray`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `Image.Image` or `np.array`. + Returns: + image ([List[Union[np.ndarray, Image.Image]]): generaited images + + """ + + # 1. Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + if guidance_scale < 1 and negative_prompt: + guidance_scale += 1 + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + # 2. Encode input prompt + text_embeddings = self._encode_prompt(prompt, do_classifier_free_guidance=do_classifier_free_guidance, negative_prompt=negative_prompt) + + # 3. Preprocess image + orig_width, orig_height = image.size + image, pad = preprocess(image, height, width) + if do_classifier_free_guidance: + image = np.concatenate(([image] * 2)) + + # 4. set timesteps + self.scheduler.set_timesteps(num_inference_steps) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = 4 + latents = self.prepare_latents( + batch_size, + num_channels_latents, + height, + width, + latents=latents, + ) + + # 6. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Expand the latents if we are doing classifier free guidance. + # The latents are expanded 3 times because for pix2pix the guidance\ + # is applied for both the text and the input image. + latent_model_input = np.concatenate( + [latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + result = self.controlnet([latent_model_input, t, text_embeddings, image], share_inputs=True, share_outputs=True) + down_and_mid_blok_samples = [sample * controlnet_conditioning_scale for _, sample in result.items()] + + # predict the noise residual + noise_pred = self.unet([latent_model_input, t, text_embeddings, *down_and_mid_blok_samples], share_inputs=True, share_outputs=True)[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[0], noise_pred[1] + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(torch.from_numpy(noise_pred), t, torch.from_numpy(latents)).prev_sample.numpy() + progress_bar.update() + + # 7. Post-processing + image = self.decode_latents(latents, pad) + + # 8. Convert to PIL + if output_type == "pil": + image = self.numpy_to_pil(image) + image = [img.resize((orig_width, orig_height), Image.Resampling.LANCZOS) for img in image] + else: + image = [cv2.resize(img, (orig_width, orig_width)) + for img in image] + + return image + + def _encode_prompt(self, prompt:Union[str, List[str]], num_images_per_prompt:int = 1, do_classifier_free_guidance:bool = True, negative_prompt:Union[str, List[str]] = None): + """ + Encodes the prompt into text encoder hidden states. + + Parameters: + prompt (str or list(str)): prompt to be encoded + num_images_per_prompt (int): number of images that should be generated per prompt + do_classifier_free_guidance (bool): whether to use classifier free guidance or not + negative_prompt (str or list(str)): negative prompt to be encoded + Returns: + text_embeddings (np.ndarray): text encoder hidden states + """ + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + # tokenize input prompts + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + + text_embeddings = self.text_encoder(text_input_ids, share_inputs=True, share_outputs=True)[0] + + # duplicate text embeddings for each generation per prompt + if num_images_per_prompt != 1: + bs_embed, seq_len, _ = text_embeddings.shape + text_embeddings = np.tile( + text_embeddings, (1, num_images_per_prompt, 1)) + text_embeddings = np.reshape( + text_embeddings, (bs_embed * num_images_per_prompt, seq_len, -1)) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_tokens: List[str] + max_length = text_input_ids.shape[-1] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + else: + uncond_tokens = negative_prompt + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + + uncond_embeddings = self.text_encoder(uncond_input.input_ids, share_inputs=True, share_outputs=True)[0] + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = uncond_embeddings.shape[1] + uncond_embeddings = np.tile(uncond_embeddings, (1, num_images_per_prompt, 1)) + uncond_embeddings = np.reshape(uncond_embeddings, (batch_size * num_images_per_prompt, seq_len, -1)) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_embeddings = np.concatenate([uncond_embeddings, text_embeddings]) + + return text_embeddings + + def prepare_latents(self, batch_size:int, num_channels_latents:int, height:int, width:int, dtype:np.dtype = torch.float32, latents:np.ndarray = None): + """ + Preparing noise to image generation. If initial latents are not provided, they will be generated randomly, + then prepared latents scaled by the standard deviation required by the scheduler + + Parameters: + batch_size (int): input batch size + num_channels_latents (int): number of channels for noise generation + height (int): image height + width (int): image width + dtype (np.dtype, *optional*, np.float32): dtype for latents generation + latents (np.ndarray, *optional*, None): initial latent noise tensor, if not provided will be generated + Returns: + latents (np.ndarray): scaled initial noise for diffusion + """ + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if latents is None: + latents = randn_tensor(shape, dtype=dtype) + else: + latents = latents + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def decode_latents(self, latents:np.array, pad:Tuple[int]): + """ + Decode predicted image from latent space using VAE Decoder and unpad image result + + Parameters: + latents (np.ndarray): image encoded in diffusion latent space + pad (Tuple[int]): each side padding sizes obtained on preprocessing step + Returns: + image: decoded by VAE decoder image + """ + latents = 1 / 0.18215 * latents + image = self.vae_decoder(latents)[0] + (_, end_h), (_, end_w) = pad[1:3] + h, w = image.shape[2:] + unpad_h = h - end_h + unpad_w = w - end_w + image = image[:, :, :unpad_h, :unpad_w] + image = np.clip(image / 2 + 0.5, 0, 1) + image = np.transpose(image, (0, 2, 3, 1)) + return image + +Prepare tokenizer and LCMScheduler +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +Tokenizer and scheduler are also important parts of the diffusion +pipeline. The tokenizer is responsible for preprocessing user-provided +prompts into token ids that then used by Text Encoder. + +The scheduler takes a model’s output (the sample which the diffusion +process is iterating on) and a timestep to return a denoised sample. The +timestep is important because it dictates where in the diffusion process +the step is; data is generated by iterating forward n timesteps and +inference occurs by propagating backward through the timesteps. There +are many +`schedulers `__ +implemented inside the diffusers library, LCM pipeline required changing +the original pipeline scheduler with +`LCMScheduler `__. + +.. code:: ipython3 + + from diffusers import LCMScheduler + from transformers import AutoTokenizer + + if not TOKENIZER_PATH.exists(): + tokenizer = AutoTokenizer.from_pretrained(stable_diffusion_id, subfolder="tokenizer") + tokenizer.save_pretrained(TOKENIZER_PATH) + else: + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH) + if not SCHEDULER_PATH.exists(): + scheduler = LCMScheduler.from_pretrained(stable_diffusion_id, subfolder="scheduler") + scheduler.save_pretrained(SCHEDULER_PATH) + else: + scheduler = LCMScheduler.from_config(SCHEDULER_PATH) + +Select inference device for Stable Diffusion pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +select device from dropdown list for running inference using OpenVINO + +.. code:: ipython3 + + import ipywidgets as widgets + + core = ov.Core() + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='CPU', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', options=('CPU', 'GPU.0', 'GPU.1', 'GPU.2', 'AUTO'), value='CPU') + + + +.. code:: ipython3 + + ov_pipe = OVControlNetStableDiffusionPipeline(tokenizer, scheduler, core, CONTROLNET_OV_PATH, TEXT_ENCODER_OV_PATH, UNET_OV_PATH, VAE_DECODER_OV_PATH, device=device.value) + +Running Text-to-Image Generation with ControlNet Conditioning and OpenVINO +-------------------------------------------------------------------------- + + + +Now, we are ready to start generation. For improving the generation +process, we also introduce an opportunity to provide a +``negative prompt``. Technically, positive prompt steers the diffusion +toward the images associated with it, while negative prompt steers the +diffusion away from it. More explanation of how it works can be found in +this +`article `__. +We can keep this field empty if we want to generate image without +negative prompting. + +`Classifier-free guidance (CFG) `__ or +guidance scale is a parameter that controls how much the image +generation process follows the text prompt. The higher the value, the +more the image sticks to a given text input. But this does not mean that +the value should always be set to maximum, as more guidance means less +diversity and quality. According to experiments, the optimal value of +guidance for LCM models is in range between 0 and 2. > Please note, that +negative prompt is applicable only when guidance scale > 1. + +Let’s see model in action + +.. code:: ipython3 + + prompt = "A head full of roses" + torch.manual_seed(4257) + + result = ov_pipe(prompt, control_image, 4) + result[0] + + + +.. parsed-literal:: + + 0%| | 0/4 [00:00`__ enables +post-training quantization by adding quantization layers into model +graph and then using a subset of the training dataset to initialize the +parameters of these additional quantization layers. Quantized operations +are executed in ``INT8`` instead of ``FP32``/``FP16`` making model +inference faster. + +According to ``OVControlNetStableDiffusionPipeline`` structure, +ControlNet and UNet are used in the cycle repeating inference on each +diffusion step, while other parts of pipeline take part only once. That +is why computation cost and speed of ControlNet and UNet become the +critical path in the pipeline. Quantizing the rest of the SD pipeline +does not significantly improve inference performance but can lead to a +substantial degradation of accuracy. + +The optimization process contains the following steps: + +1. Create a calibration dataset for quantization. +2. Run ``nncf.quantize()`` to obtain quantized model. +3. Save the ``INT8`` model using ``openvino.save_model()`` function. + +Please select below whether you would like to run quantization to +improve model inference speed. + +.. code:: ipython3 + + is_gpu_device = "GPU" in device.value + to_quantize = widgets.Checkbox( + value=not is_gpu_device, + description='Quantization', + disabled=is_gpu_device, + ) + + to_quantize + +Let’s load ``skip magic`` extension to skip quantization if +``to_quantize`` is not selected + +.. code:: ipython3 + + import sys + sys.path.append("../utils") + + int8_pipe = None + + %load_ext skip_kernel_extension + +Prepare calibration datasets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +We use a portion of +`fusing/instructpix2pix-1000-samples `__ +dataset from Hugging Face as calibration data for ControlNet and UNet. + +To collect intermediate model inputs for calibration we should customize +``CompiledModel``. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + import datasets + from tqdm.notebook import tqdm + from transformers import set_seed + from typing import Any, Dict, List + + set_seed(1) + + class CompiledModelDecorator(ov.CompiledModel): + def __init__(self, compiled_model, prob: float): + super().__init__(compiled_model) + self.data_cache = [] + self.prob = np.clip(prob, 0, 1) + + def __call__(self, *args, **kwargs): + if np.random.rand() >= self.prob: + self.data_cache.append(*args) + return super().__call__(*args, **kwargs) + + def collect_calibration_data(pipeline: OVControlNetStableDiffusionPipeline, subset_size: int) -> List[Dict]: + original_unet = pipeline.unet + pipeline.unet = CompiledModelDecorator(original_unet, prob=0.3) + + dataset = datasets.load_dataset("fusing/instructpix2pix-1000-samples", split="train", streaming=True).shuffle(seed=42) + pipeline.set_progress_bar_config(disable=True) + + # Run inference for data collection + pbar = tqdm(total=subset_size) + diff = 0 + control_images = [] + for batch in dataset: + prompt = batch["edit_prompt"] + if len(prompt) > tokenizer.model_max_length: + continue + image = batch["input_image"] + control_image = processor(image) + + _ = pipeline(prompt, image=control_image, num_inference_steps=4) + collected_subset_size = len(pipeline.unet.data_cache) + control_images.append((min(collected_subset_size, subset_size), control_image)) + if collected_subset_size >= subset_size: + pbar.update(subset_size - pbar.n) + break + pbar.update(collected_subset_size - diff) + diff = collected_subset_size + + control_calibration_dataset = pipeline.unet.data_cache + pipeline.set_progress_bar_config(disable=False) + pipeline.unet = original_unet + return control_calibration_dataset, control_images + +.. code:: ipython3 + + %%skip not $to_quantize.value + + CONTROLNET_INT8_OV_PATH = Path("model/controlnet-normalbae_int8.xml") + UNET_INT8_OV_PATH = Path("model/unet_controlnet_int8.xml") + if not (CONTROLNET_INT8_OV_PATH.exists() and UNET_INT8_OV_PATH.exists()): + subset_size = 200 + unet_calibration_data, control_images = collect_calibration_data(ov_pipe, subset_size=subset_size) + + + +.. parsed-literal:: + + 0%| | 0/200 [00:00 +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/263-lcm-lora-controlnet-with-output_files/ + +

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/263-lcm-lora-controlnet-with-output_files/


../
+263-lcm-lora-controlnet-with-output_10_1.png       07-Dec-2023 00:49             2088439
+263-lcm-lora-controlnet-with-output_27_2.jpg       07-Dec-2023 00:49               51043
+263-lcm-lora-controlnet-with-output_27_2.png       07-Dec-2023 00:49              479182
+263-lcm-lora-controlnet-with-output_28_0.png       07-Dec-2023 00:49             1877881
+263-lcm-lora-controlnet-with-output_42_1.png       07-Dec-2023 00:49             2501299
+

+ diff --git a/docs/notebooks/265-wuerstchen-image-generation-with-output.rst b/docs/notebooks/265-wuerstchen-image-generation-with-output.rst index 9e1e9e7362f..1b09419feb6 100644 --- a/docs/notebooks/265-wuerstchen-image-generation-with-output.rst +++ b/docs/notebooks/265-wuerstchen-image-generation-with-output.rst @@ -28,6 +28,7 @@ Hub `__. **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Load the original model <#load-the-original-model>`__ @@ -51,9 +52,7 @@ Prerequisites .. code:: ipython3 - %pip install -q "diffusers>=0.21.0" transformers accelerate matplotlib gradio - %pip uninstall -q -y openvino-dev openvino openvino-nightly - %pip install -q openvino-nightly + %pip install -q "diffusers>=0.21.0" transformers accelerate matplotlib gradio "openvino>=2023.2.0" --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: diff --git a/docs/notebooks/265-wuerstchen-image-generation-with-output_files/index.html b/docs/notebooks/265-wuerstchen-image-generation-with-output_files/index.html index c37effc83e1..b1135c65f26 100644 --- a/docs/notebooks/265-wuerstchen-image-generation-with-output_files/index.html +++ b/docs/notebooks/265-wuerstchen-image-generation-with-output_files/index.html @@ -1,8 +1,8 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/265-wuerstchen-image-generation-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/265-wuerstchen-image-generation-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/265-wuerstchen-image-generation-with-output_files/


../
-265-wuerstchen-image-generation-with-output_11_..> 15-Nov-2023 00:43             1321476
-265-wuerstchen-image-generation-with-output_45_..> 15-Nov-2023 00:43             1293108
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/265-wuerstchen-image-generation-with-output_files/


../
+265-wuerstchen-image-generation-with-output_11_..> 07-Dec-2023 00:49             1321476
+265-wuerstchen-image-generation-with-output_45_..> 07-Dec-2023 00:49             1293108
 

diff --git a/docs/notebooks/266-speculative-sampling-with-output.rst b/docs/notebooks/266-speculative-sampling-with-output.rst index 42f694e151a..a3140067703 100644 --- a/docs/notebooks/266-speculative-sampling-with-output.rst +++ b/docs/notebooks/266-speculative-sampling-with-output.rst @@ -35,6 +35,7 @@ available at openvino.ai **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Select inference device <#select-inference-device>`__ @@ -71,7 +72,7 @@ useful modules. .. code:: ipython3 %pip install -q --upgrade pip - %pip install -q --upgrade transformers torch gradio openvino accelerate onnx onnxruntime ipywidgets + %pip install -q --upgrade transformers torch gradio openvino accelerate onnx ipywidgets --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/huggingface/optimum-intel.git" Select inference device diff --git a/docs/notebooks/267-distil-whisper-asr-with-output.rst b/docs/notebooks/267-distil-whisper-asr-with-output.rst index f9fc4e58cb4..3378adcca8d 100644 --- a/docs/notebooks/267-distil-whisper-asr-with-output.rst +++ b/docs/notebooks/267-distil-whisper-asr-with-output.rst @@ -7,9 +7,9 @@ is a distilled variant of the OpenAI. The Distil-Whisper is proposed in the paper `Robust Knowledge Distillation via Large-Scale Pseudo Labelling `__. According to authors, -compared to Whisper, Distil-Whisper runs 6x faster with 50% fewer -parameters, while performing to within 1% word error rate (WER) on -out-of-distribution evaluation data. +compared to Whisper, Distil-Whisper runs in several times faster with +50% fewer parameters, while performing to within 1% word error rate +(WER) on out-of-distribution evaluation data. Whisper is a Transformer based encoder-decoder model, also referred to as a sequence-to-sequence model. It maps a sequence of audio spectrogram @@ -57,23 +57,17 @@ from `NNCF `__ is applied. - `Interactive demo <#interactive-demo>`__ Prerequisites -------------------------------------------------------- +------------- + + .. code:: ipython3 - %pip uninstall -q -y openvino-dev openvino openvino-nightly - %pip install -q openvino-nightly - %pip install -q "transformers" onnx datasets "git+https://github.com/eaidova/optimum-intel.git@ea/whisper" "gradio>=4.0" "librosa" "soundfile" + %pip uninstall -q -y optimum-intel optimum + %pip install -q transformers onnx "git+https://github.com/eaidova/optimum-intel.git@ea/whisper" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "openvino>=2023.2.0" datasets "gradio>=4.0" "librosa" "soundfile" %pip install -q "nncf>=2.6.0" "jiwer" - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - ERROR: tokenizers 0.14.1 has requirement huggingface_hub<0.18,>=0.16.4, but you'll have huggingface-hub 0.19.0 which is incompatible. - Note: you may need to restart the kernel to use updated packages. - - Load PyTorch model ------------------ @@ -238,8 +232,10 @@ OpenVINO model. It means that we can reuse initialized early processor. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino -Select Inference device\ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select Inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -267,7 +263,9 @@ Select Inference device\ Compile OpenVINO model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -283,7 +281,9 @@ Compile OpenVINO model Run OpenVINO model inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -320,8 +320,10 @@ Run OpenVINO model inference Result: Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel. -Compare performance PyTorch vs OpenVINO\ ---------------------------------------------------------------------------------- +Compare performance PyTorch vs OpenVINO +--------------------------------------- + + .. code:: ipython3 @@ -375,7 +377,9 @@ Compare performance PyTorch vs OpenVINO\ Compare with OpenAI Whisper -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Since Distil-Whisper is optimized version of original OpenAI Whisper model, let’s compare performance and check benefits of using it. @@ -1225,4 +1229,3 @@ recognition. Multilingual support will be provided later. ..
- diff --git a/docs/notebooks/268-table-question-answering-with-output.rst b/docs/notebooks/268-table-question-answering-with-output.rst index 60a6495dbd0..e26f3006198 100644 --- a/docs/notebooks/268-table-question-answering-with-output.rst +++ b/docs/notebooks/268-table-question-answering-with-output.rst @@ -30,6 +30,7 @@ and is trained end-to-end. **Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Use the original model to run an inference <#use-the-original-model-to-run-an-inference>`__ @@ -46,19 +47,13 @@ Prerequisites .. code:: ipython3 - %pip uninstall -q -y openvino-dev openvino openvino-nightly - %pip install -q openvino-nightly - # other dependencies + %pip install -q torch "transformers>=4.31.0" --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "gradio>=4.0.2" + %pip install -q "openvino>=2023.2.0" "gradio>=4.0.2" .. parsed-literal:: - WARNING: Skipping openvino-nightly as it is not installed. - Note: you may need to restart the kernel to use updated packages. - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - Note: you may need to restart the kernel to use updated packages. DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 @@ -76,10 +71,10 @@ Prerequisites .. parsed-literal:: - 2023-11-15 00:16:22.014004: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-11-15 00:16:22.047161: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-07 00:22:18.058770: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-07 00:22:18.092302: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-11-15 00:16:22.631876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-07 00:22:18.675506: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Use ``TapasForQuestionAnswering.from_pretrained`` to download a @@ -169,7 +164,7 @@ demonstrate how to make an inference. You can use ``pipeline`` from .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1785: UserWarning: scatter_reduce() is in beta and the API may change at any time. (Triggered internally at ../aten/src/ATen/native/TensorAdvancedIndexing.cpp:1615.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1785: UserWarning: scatter_reduce() is in beta and the API may change at any time. (Triggered internally at ../aten/src/ATen/native/TensorAdvancedIndexing.cpp:1615.) segment_means = out.scatter_reduce( @@ -221,55 +216,55 @@ function to serialize the result of conversion. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1600: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1600: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. self.indices = torch.as_tensor(indices) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1601: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1601: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. self.num_segments = torch.as_tensor(num_segments, device=indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1703: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1703: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. batch_size = torch.prod(torch.tensor(list(index.batch_shape()))) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1779: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1779: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. [torch.as_tensor([-1], dtype=torch.long), torch.as_tensor(vector_shape, dtype=torch.long)], dim=0 - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1782: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1782: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! flat_values = values.reshape(flattened_shape.tolist()) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1784: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1784: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! out = torch.zeros(int(flat_index.num_segments), dtype=torch.float, device=flat_values.device) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1792: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1792: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(index.batch_shape(), dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1793: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1793: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor([index.num_segments], dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1794: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1794: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(vector_shape, dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1799: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1799: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! output_values = segment_means.clone().view(new_shape.tolist()).to(values.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1730: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1730: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. batch_shape = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1734: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1734: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. num_segments = torch.as_tensor(num_segments) # create a rank 0 tensor (scalar) containing num_segments (e.g. 64) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1745: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1745: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! new_shape = [int(x) for x in new_tensor.tolist()] - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1748: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1748: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. multiples = torch.cat([batch_shape, torch.as_tensor([1])], dim=0) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1749: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1749: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! indices = indices.repeat(multiples.tolist()) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:316: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:316: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(self.config.max_position_embeddings - 1, device=device), position - first_position - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1260: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1260: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. indices=torch.min(row_ids, torch.as_tensor(self.config.max_num_rows - 1, device=row_ids.device)), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1265: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1265: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. indices=torch.min(column_ids, torch.as_tensor(self.config.max_num_columns - 1, device=column_ids.device)), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1957: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1957: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1962: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1962: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1998: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1998: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. labels_per_column, _ = reduce_sum(torch.as_tensor(labels, dtype=torch.float32, device=labels.device), col_index) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2021: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2021: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(labels, dtype=torch.long, device=labels.device), cell_index - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2028: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2028: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_mask = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2053: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2053: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. selected_column_id = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2058: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2058: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. selected_column_mask = torch.as_tensor( @@ -461,14 +456,6 @@ Interactive inference demo.queue().launch(share=True, debug=False) -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/gradio/blocks.py:928: UserWarning: api_name display_table already exists, using display_table_1 - warnings.warn(f"api_name {api_name} already exists, using {api_name_}") - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-545/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/gradio/blocks.py:928: UserWarning: api_name display_table already exists, using display_table_2 - warnings.warn(f"api_name {api_name} already exists, using {api_name_}") - - .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 diff --git a/docs/notebooks/269-film-slowmo-with-output.rst b/docs/notebooks/269-film-slowmo-with-output.rst index 14841979ccc..9cee0960e9c 100644 --- a/docs/notebooks/269-film-slowmo-with-output.rst +++ b/docs/notebooks/269-film-slowmo-with-output.rst @@ -34,22 +34,34 @@ FILM is trained on regular video frame triplets, with the middle frame serving as the ground-truth for supervision. In this tutorial, we will use `TensorFlow Hub `__ as -a model source. +a model source. + + **NOTE**: To run this tutorial, your system is required to have a VP9 + video encoder. Ubuntu has it preinstalled, but for Windows, you + should install it manually. **Table of contents:** -- `Prerequisites <#prerequisites>`__ -- `Prepare images <#prepare-images>`__ -- `Load the model <#load-the-model>`__ -- `Infer the model <#infer-the-model>`__ -- `Single middle frame interpolation <#single-middle-frame-interpolation>`__ -- `Recursive frame generation <#recursive-frame-generation>`__ -- `Convert the model to OpenVINO IR <#convert-the-model-to-openvino-ir>`__ -- `Inference <#inference>`__ -- `Select inference device <#select-inference-device>`__ -- `Single middle frame interpolation <#single-middle-frame-interpolation>`__ -- `Recursive frame generation <#recursive-frame-generation>`__ -- `Interactive inference <#interactive-inference>`__ + +- `Prerequisites <#prerequisites>`__ +- `Prepare images <#prepare-images>`__ +- `Load the model <#load-the-model>`__ +- `Infer the model <#infer-the-model>`__ + + - `Single middle frame + interpolation <#single-middle-frame-interpolation>`__ + - `Recursive frame generation <#recursive-frame-generation>`__ + +- `Convert the model to OpenVINO + IR <#convert-the-model-to-openvino-ir>`__ +- `Inference <#inference>`__ + + - `Select inference device <#select-inference-device>`__ + - `Single middle frame + interpolation <#single-middle-frame-interpolation>`__ + - `Recursive frame generation <#recursive-frame-generation>`__ + +- `Interactive inference <#interactive-inference>`__ .. |image0| image:: https://github.com/googlestaging/frame-interpolation/raw/main/moment.gif @@ -60,9 +72,7 @@ Prerequisites .. code:: ipython3 - %pip install -q tensorflow tensorflow_hub numpy "opencv-python" tqdm matplotlib gradio Pillow - %pip uninstall -q -y openvino-dev openvino openvino-nightly - %pip install -q openvino-nightly + %pip install -q tensorflow tensorflow_hub numpy "opencv-python" tqdm matplotlib gradio Pillow "openvino>=2023.2.0" .. parsed-literal:: @@ -201,7 +211,8 @@ Hub `__. x1=tf.keras.layers.Input(shape=(None, None, 3)), time=tf.keras.layers.Input(shape=(1)), ) - film_layer = hub.KerasLayer("https://tfhub.dev/google/film/1")(inputs) + model_url = "https://www.kaggle.com/models/google/film/frameworks/tensorFlow2/variations/film/versions/1" + film_layer = hub.KerasLayer(model_url)(inputs) film_model = tf.keras.Model(inputs=inputs, outputs=film_layer) Infer the model diff --git a/docs/notebooks/269-film-slowmo-with-output_files/index.html b/docs/notebooks/269-film-slowmo-with-output_files/index.html index 48822d60d20..d6b28a7018d 100644 --- a/docs/notebooks/269-film-slowmo-with-output_files/index.html +++ b/docs/notebooks/269-film-slowmo-with-output_files/index.html @@ -1,9 +1,9 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/269-film-slowmo-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/269-film-slowmo-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231114220808/dist/rst_files/269-film-slowmo-with-output_files/


../
-269-film-slowmo-with-output_14_0.png               15-Nov-2023 00:43              550928
-269-film-slowmo-with-output_29_0.png               15-Nov-2023 00:43              550615
-269-film-slowmo-with-output_7_0.png                15-Nov-2023 00:43              794884
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/269-film-slowmo-with-output_files/


../
+269-film-slowmo-with-output_14_0.png               07-Dec-2023 00:49              550928
+269-film-slowmo-with-output_29_0.png               07-Dec-2023 00:49              550615
+269-film-slowmo-with-output_7_0.png                07-Dec-2023 00:49              794884
 

diff --git a/docs/notebooks/270-sound-generation-audioldm2-with-output.rst b/docs/notebooks/270-sound-generation-audioldm2-with-output.rst new file mode 100644 index 00000000000..4999c60715a --- /dev/null +++ b/docs/notebooks/270-sound-generation-audioldm2-with-output.rst @@ -0,0 +1,774 @@ +Sound Generation with AudioLDM2 and OpenVINO™ +============================================= + +`AudioLDM 2 `__ is a latent +text-to-audio diffusion model capable of generating realistic audio +samples given any text input. + +AudioLDM 2 was proposed in the paper `AudioLDM 2: Learning Holistic +Audio Generation with Self-supervised +Pretraining `__ by ``Haohe Liu`` et +al. + +The model takes a text prompt as input and predicts the corresponding +audio. It can generate text-conditional sound effects, human speech and +music. + +|image0| + +In this tutorial we will try out the pipeline, convert the models +backing it one by one and will run an interactive app with Gradio! + +**Table of contents:** + + +- `Prerequisites <#prerequisites>`__ +- `Instantiating Generation + Pipeline <#instantiating-generation-pipeline>`__ +- `Convert models to OpenVINO Intermediate representation (IR) + format <#convert-models-to-openvino-intermediate-representation-ir-format>`__ + + - `Text Encoder <#text-encoder>`__ + - `Second text encoder + conversion <#second-text-encoder-conversion>`__ + - `Vocoder conversion <#vocoder-conversion>`__ + - `GPT-2 conversion <#gpt--conversion>`__ + - `Projection model conversion <#projection-model-conversion>`__ + - `UNet conversion <#unet-conversion>`__ + - `VAE Decoder conversion <#vae-decoder-conversion>`__ + +- `Select inference device for Stable Diffusion + pipeline <#select-inference-device-for-stable-diffusion-pipeline>`__ +- `Adapt OpenVINO models to the original + pipeline <#adapt-openvino-models-to-the-original-pipeline>`__ +- `Try out the converted pipeline <#try-out-the-converted-pipeline>`__ + +.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/76463150/c93a0f86-d9cf-4bd1-93b9-e27532170d75 + +Prerequisites +------------- + + + +.. code:: ipython3 + + %pip install -q accelerate "diffusers>=0.21.0" transformers torch gradio --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "openvino>=2023.2.0" + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + + +Instantiating Generation Pipeline +--------------------------------- + + + +To work with `AudioLDM 2 `__ by +`Centre for Vision, Speech and Signal Processing - University of Surrey `__, +we will use `Hugging Face Diffusers +package `__. Diffusers package +exposes the ``AudioLDM2Pipeline`` class, simplifying the model +instantiation and weights loading. The code below demonstrates how to +create a ``AudioLDM2Pipeline`` and generate a text-conditioned sound +sample. + +.. code:: ipython3 + + from collections import namedtuple + from functools import partial + import gc + from pathlib import Path + + from diffusers import AudioLDM2Pipeline + from IPython.display import Audio + import numpy as np + import openvino as ov + import torch + + MODEL_ID = "cvssp/audioldm2" + pipe = AudioLDM2Pipeline.from_pretrained(MODEL_ID) + + prompt = "birds singing in the forest" + negative_prompt = "Low quality" + audio = pipe( + prompt, + negative_prompt=negative_prompt, + num_inference_steps=150, + audio_length_in_s=3.0 + ).audios[0] + + sampling_rate = 16000 + Audio(audio, rate=sampling_rate) + + + +.. parsed-literal:: + + Loading pipeline components...: 0%| | 0/11 [00:00 + + Your browser does not support the audio element. + + + + + +Convert models to OpenVINO Intermediate representation (IR) format +------------------------------------------------------------------ + + + +`Model conversion +API `__ +enables direct conversion of PyTorch models backing the pipeline. We +need to provide a model object, input data for model tracing to +``ov.convert_model`` function to obtain OpenVINO ``ov.Model`` object +instance. Model can be saved on disk for next deployment using +``ov.save_model`` function. + +The pipeline consists of seven important parts: + +- T5 and CLAP Text Encoders for creation condition to generate an sound + from a text prompt. +- Projection model to merge outputs from the two text encoders. +- GPT-2 language model to generate a sequence of hidden-states + conditioned on the projected outputs from the two text encoders. +- Vocoder to convert the mel-spectrogram latents to the final audio + waveform. +- Unet for step-by-step denoising latent image representation. +- Autoencoder (VAE) for decoding latent space to image. + +.. code:: ipython3 + + models_base_folder = Path("models") + + def cleanup_torchscript_cache(): + """ + Helper for removing cached model representation + """ + torch._C._jit_clear_class_registry() + torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() + torch.jit._state._clear_class_state() + +CLAP Text Encoder Conversion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +First frozen text-encoder. AudioLDM2 uses the joint audio-text embedding +model +`CLAP `__, +specifically the +`laion/clap-htsat-unfused `__ +variant. The text branch is used to encode the text prompt to a prompt +embedding. The full audio-text model is used to rank generated waveforms +against the text prompt by computing similarity scores. + +.. code:: ipython3 + + class ClapEncoderWrapper(torch.nn.Module): + def __init__(self, encoder): + super().__init__() + encoder.eval() + self.encoder = encoder + + def forward(self, input_ids, attention_mask): + return self.encoder.get_text_features(input_ids, attention_mask) + + clap_text_encoder_ir_path = models_base_folder / "clap_text_encoder.xml" + + if not clap_text_encoder_ir_path.exists(): + with torch.no_grad(): + ov_model = ov.convert_model( + ClapEncoderWrapper(pipe.text_encoder), # model instance + example_input={ + "input_ids": torch.ones((1, 512), dtype=torch.long), + "attention_mask": torch.ones((1, 512), dtype=torch.long), + }, # inputs for model tracing + ) + ov.save_model(ov_model, clap_text_encoder_ir_path) + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("Text Encoder successfully converted to IR") + else: + print(f"Text Encoder will be loaded from {clap_text_encoder_ir_path}") + + +.. parsed-literal:: + + Text Encoder will be loaded from clap_text_encoder.xml + + +T5 Text Encoder Conversion +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +As second frozen text-encoder, AudioLDM2 uses the +`T5 `__, +specifically the +`google/flan-t5-large `__ +variant. + +The text-encoder is responsible for transforming the input prompt, for +example, “birds singing in the forest” into an embedding space that can +be understood by the U-Net. It is usually a simple transformer-based +encoder that maps a sequence of input tokens to a sequence of latent +text embeddings. + +The input of the text encoder is tensor ``input_ids``, which contains +indexes of tokens from text processed by the tokenizer and padded to the +maximum length accepted by the model. Model outputs are two tensors: +``last_hidden_state`` - hidden state from the last MultiHeadAttention +layer in the model and ``pooler_out`` - pooled output for whole model +hidden states. + +.. code:: ipython3 + + t5_text_encoder_ir_path = models_base_folder / "t5_text_encoder.xml" + + if not t5_text_encoder_ir_path.exists(): + pipe.text_encoder_2.eval() + with torch.no_grad(): + ov_model = ov.convert_model( + pipe.text_encoder_2, # model instance + example_input=torch.ones((1, 7), dtype=torch.long), # inputs for model tracing + ) + ov.save_model(ov_model, t5_text_encoder_ir_path) + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("Text Encoder successfully converted to IR") + else: + print(f"Text Encoder will be loaded from {t5_text_encoder_ir_path}") + + +.. parsed-literal:: + + Text Encoder will be loaded from t5_text_encoder.xml + + +Projection model conversion +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +A trained model used to linearly project the hidden-states from the +first and second text encoder models and insert learned Start Of +Sequence and End Of Sequence token embeddings. The projected +hidden-states from the two text encoders are concatenated to give the +input to the language model. + +.. code:: ipython3 + + projection_model_ir_path = models_base_folder / "projection_model.xml" + + projection_model_inputs = { + "hidden_states": torch.randn((1, 1, 512), dtype=torch.float32), + "hidden_states_1": torch.randn((1, 7, 1024), dtype=torch.float32), + "attention_mask": torch.ones((1, 1), dtype=torch.int64), + "attention_mask_1": torch.ones((1, 7), dtype=torch.int64), + } + + if not projection_model_ir_path.exists(): + pipe.projection_model.eval() + with torch.no_grad(): + ov_model = ov.convert_model( + pipe.projection_model, # model instance + example_input=projection_model_inputs, # inputs for model tracing + ) + ov.save_model(ov_model, projection_model_ir_path) + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("The Projection Model successfully converted to IR") + else: + print(f"The Projection Model will be loaded from {projection_model_ir_path}") + + +.. parsed-literal:: + + The Projection Model will be loaded from projection_model.xml + + +GPT-2 conversion +~~~~~~~~~~~~~~~~ + + + +`GPT-2 `__ is an auto-regressive language +model used to generate a sequence of hidden-states conditioned on the +projected outputs from the two text encoders. + +.. code:: ipython3 + + language_model_ir_path = models_base_folder / "language_model.xml" + + language_model_inputs = { + "inputs_embeds": torch.randn((1, 12, 768), dtype=torch.float32), + "attention_mask": torch.ones((1, 12), dtype=torch.int64), + } + + if not language_model_ir_path.exists(): + pipe.language_model.config.torchscript = True + pipe.language_model.eval() + pipe.language_model.__call__ = partial(pipe.language_model.__call__, kwargs={ + "past_key_values": None, + "use_cache": False, + "return_dict": False}) + with torch.no_grad(): + ov_model = ov.convert_model( + pipe.language_model, # model instance + example_input=language_model_inputs, # inputs for model tracing + ) + + ov_model.inputs[0].get_node().set_partial_shape(ov.PartialShape([1, -1])) + ov_model.inputs[0].get_node().set_element_type(ov.Type.i64) + ov_model.inputs[1].get_node().set_partial_shape(ov.PartialShape([1, -1, 768])) + ov_model.inputs[1].get_node().set_element_type(ov.Type.f32) + + ov_model.validate_nodes_and_infer_types() + + ov.save_model(ov_model, language_model_ir_path) + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("The Projection Model successfully converted to IR") + else: + print(f"The Projection Model will be loaded from {language_model_ir_path}") + + +.. parsed-literal:: + + The Projection Model will be loaded from language_model.xml + + +Vocoder conversion +~~~~~~~~~~~~~~~~~~ + + + +`SpeechT5 HiFi-GAN Vocoder `__ +is used to convert the mel-spectrogram latents to the final audio +waveform. + +.. code:: ipython3 + + vocoder_ir_path = models_base_folder / "vocoder.xml" + + if not vocoder_ir_path.exists(): + pipe.vocoder.eval() + with torch.no_grad(): + ov_model = ov.convert_model( + pipe.vocoder, # model instance + example_input=torch.ones((1, 700, 64), dtype=torch.float32), # inputs for model tracing + ) + ov.save_model(ov_model, vocoder_ir_path) + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("The Vocoder successfully converted to IR") + else: + print(f"The Vocoder will be loaded from {vocoder_ir_path}") + + +.. parsed-literal:: + + The Vocoder will be loaded from vocoder.xml + + +UNet conversion +~~~~~~~~~~~~~~~ + + + +The UNet model is used to denoise the encoded audio latents. The process +of UNet model conversion remains the same, like for original Stable +Diffusion model. + +.. code:: ipython3 + + unet_ir_path = models_base_folder / "unet.xml" + + pipe.unet.eval() + unet_inputs = { + "sample": torch.randn((2, 8, 75, 16), dtype=torch.float32), + "timestep": torch.tensor(1, dtype=torch.int64), + "encoder_hidden_states": torch.randn((2, 8, 768), dtype=torch.float32), + "encoder_hidden_states_1": torch.randn((2, 7, 1024), dtype=torch.float32), + "encoder_attention_mask_1": torch.ones((2, 7), dtype=torch.int64), + } + + if not unet_ir_path.exists(): + with torch.no_grad(): + ov_model = ov.convert_model(pipe.unet, example_input=unet_inputs) + + ov_model.inputs[0].get_node().set_partial_shape(ov.PartialShape((2, 8, -1, 16))) + ov_model.inputs[2].get_node().set_partial_shape(ov.PartialShape((2, 8, 768))) + ov_model.inputs[3].get_node().set_partial_shape(ov.PartialShape((2, -1, 1024))) + ov_model.inputs[4].get_node().set_partial_shape(ov.PartialShape((2, -1))) + ov_model.validate_nodes_and_infer_types() + + ov.save_model(ov_model, unet_ir_path) + + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("Unet successfully converted to IR") + else: + print(f"Unet will be loaded from {unet_ir_path}") + + +.. parsed-literal:: + + Unet will be loaded from unet.xml + + +VAE Decoder conversion +~~~~~~~~~~~~~~~~~~~~~~ + + + +The VAE model has two parts, an encoder, and a decoder. The encoder is +used to convert the image into a low-dimensional latent representation, +which will serve as the input to the U-Net model. The decoder, +conversely, transforms the latent representation back into an image. + +During latent diffusion training, the encoder is used to get the latent +representations (latents) of the images for the forward diffusion +process, which applies more and more noise at each step. During +inference, the denoised latents generated by the reverse diffusion +process are converted back into images using the VAE decoder. During +inference, we will see that we **only need the VAE decoder**. You can +find instructions on how to convert the encoder part in a stable +diffusion +`notebook <225-stable-diffusion-text-to-image-with-output.html>`__. + +.. code:: ipython3 + + vae_ir_path = models_base_folder / "vae.xml" + + class VAEDecoderWrapper(torch.nn.Module): + def __init__(self, vae): + super().__init__() + vae.eval() + self.vae = vae + + def forward(self, latents): + return self.vae.decode(latents) + + if not vae_ir_path.exists(): + vae_decoder = VAEDecoderWrapper(pipe.vae) + latents = torch.zeros((1, 8, 175, 16)) + + vae_decoder.eval() + with torch.no_grad(): + ov_model = ov.convert_model(vae_decoder, example_input=latents) + ov.save_model(ov_model, vae_ir_path) + del ov_model + cleanup_torchscript_cache() + gc.collect() + print("VAE decoder successfully converted to IR") + else: + print(f"VAE decoder will be loaded from {vae_ir_path}") + + +.. parsed-literal:: + + VAE decoder will be loaded from vae.xml + + +Select inference device for Stable Diffusion pipeline +----------------------------------------------------- + + + +select device from dropdown list for running inference using OpenVINO + +.. code:: ipython3 + + import ipywidgets as widgets + + core = ov.Core() + + DEVICE = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value="CPU", + description="Device:", + disabled=False, + ) + + DEVICE + + + + +.. parsed-literal:: + + Dropdown(description='Device:', options=('CPU', 'AUTO'), value='CPU') + + + +Adapt OpenVINO models to the original pipeline +---------------------------------------------- + + + +Here we create wrapper classes for all three OpenVINO models that we +want to embed in the original inference pipeline. Here are some of the +things to consider when adapting an OV model: - Make sure that +parameters passed by the original pipeline are forwarded to the compiled +OV model properly; sometimes the OV model uses only a portion of the +input arguments and some are ignored, sometimes you need to convert the +argument to another data type or unwrap some data structures such as +tuples or dictionaries. - Do guarantee that the wrapper class returns +results to the pipeline in an expected format. In the example below you +can see how we pack OV model outputs into special named tuples to adapt +them for the pipeline. - Pay attention to the model method used in the +original pipeline for calling the model - it may be not the ``forward`` +method! Refer to the ``OVClapEncoderWrapper`` to see how we wrap OV +model inference into the ``get_text_features`` method. + +.. code:: ipython3 + + class OVClapEncoderWrapper: + def __init__(self, encoder_ir, config): + self.encoder = core.compile_model(encoder_ir, DEVICE.value) + self.config = config + + def get_text_features(self, input_ids, attention_mask, **_): + last_hidden_state = self.encoder([input_ids, attention_mask])[0] + return torch.from_numpy(last_hidden_state) + + class OVT5EncoderWrapper: + def __init__(self, encoder_ir, config): + self.encoder = core.compile_model(encoder_ir, DEVICE.value) + self.config = config + self.dtype = self.config.torch_dtype + + def __call__(self, input_ids, **_): + last_hidden_state = self.encoder(input_ids)[0] + return torch.from_numpy(last_hidden_state)[None, ...] + + class OVVocoderWrapper: + def __init__(self, vocoder_ir, config): + self.vocoder = core.compile_model(vocoder_ir, DEVICE.value) + self.config = config + + def __call__(self, mel_spectrogram, **_): + waveform = self.vocoder(mel_spectrogram)[0] + return torch.from_numpy(waveform) + + class OVProjectionModelWrapper: + def __init__(self, proj_model_ir, config): + self.proj_model = core.compile_model(proj_model_ir, DEVICE.value) + self.config = config + self.output_type = namedtuple("ProjectionOutput", ["hidden_states", "attention_mask"]) + + def __call__( + self, hidden_states, + hidden_states_1, + attention_mask, + attention_mask_1, **_ + ): + output = self.proj_model({ + "hidden_states": hidden_states, + "hidden_states_1": hidden_states_1, + "attention_mask": attention_mask, + "attention_mask_1": attention_mask_1, + }) + return self.output_type(torch.from_numpy(output[0]), torch.from_numpy(output[1])) + + class OVUnetWrapper: + def __init__(self, unet_ir, config): + self.unet = core.compile_model(unet_ir, DEVICE.value) + self.config = config + + def __call__( + self, sample, + timestep, + encoder_hidden_states, + encoder_hidden_states_1, + encoder_attention_mask_1, **_ + ): + output = self.unet({ + "sample": sample, + "timestep": timestep, + "encoder_hidden_states": encoder_hidden_states, + "encoder_hidden_states_1": encoder_hidden_states_1, + "encoder_attention_mask_1": encoder_attention_mask_1, + }) + return (torch.from_numpy(output[0]), ) + + class OVVaeDecoderWrapper: + def __init__(self, vae_ir, config): + self.vae = core.compile_model(vae_ir, DEVICE.value) + self.config = config + self.output_type = namedtuple("VaeOutput", ["sample"]) + + def decode(self, latents, **_): + last_hidden_state = self.vae(latents)[0] + return self.output_type(torch.from_numpy(last_hidden_state)) + + def generate_language_model( + gpt_2: ov.CompiledModel, + inputs_embeds: torch.Tensor, + attention_mask: torch.Tensor, + max_new_tokens: int = 8, + **_ + ) -> torch.Tensor: + """ + Generates a sequence of hidden-states from the language model, conditioned on the embedding inputs. + """ + if not max_new_tokens: + max_new_tokens = 8 + inputs_embeds = inputs_embeds.cpu().numpy() + attention_mask = attention_mask.cpu().numpy() + for _ in range(max_new_tokens): + # forward pass to get next hidden states + output = gpt_2({"inputs_embeds":inputs_embeds, "attention_mask":attention_mask}) + + next_hidden_states = output[0] + + # Update the model input + inputs_embeds = np.concatenate([inputs_embeds, next_hidden_states[:, -1:, :]], axis=1) + attention_mask = np.concatenate([attention_mask, np.ones((attention_mask.shape[0], 1))], axis=1) + return torch.from_numpy(inputs_embeds[:, -max_new_tokens:, :]) + + +Now we initialize the wrapper objects and load them to the HF pipeline + +.. code:: ipython3 + + pipe = AudioLDM2Pipeline.from_pretrained(MODEL_ID) + pipe.config.torchscript = True + pipe.config.return_dict = False + + np.random.seed(0) + torch.manual_seed(0) + + pipe.text_encoder = OVClapEncoderWrapper(clap_text_encoder_ir_path, pipe.text_encoder.config) + pipe.text_encoder_2 = OVT5EncoderWrapper(t5_text_encoder_ir_path, pipe.text_encoder_2.config) + pipe.projection_model = OVProjectionModelWrapper(projection_model_ir_path, pipe.projection_model.config) + pipe.vocoder = OVVocoderWrapper(vocoder_ir_path, pipe.vocoder.config) + pipe.unet = OVUnetWrapper(unet_ir_path, pipe.unet.config) + pipe.vae = OVVaeDecoderWrapper(vae_ir_path, pipe.vae.config) + + pipe.generate_language_model = partial(generate_language_model, core.compile_model(language_model_ir_path, DEVICE.value)) + + gc.collect() + + prompt = "birds singing in the forest" + negative_prompt = "Low quality" + audio = pipe( + prompt, + negative_prompt=negative_prompt, + num_inference_steps=150, + audio_length_in_s=3.0 + ).audios[0] + + sampling_rate = 16000 + Audio(audio, rate=sampling_rate) + + + +.. parsed-literal:: + + Loading pipeline components...: 0%| | 0/11 [00:00 + + Your browser does not support the audio element. + + + + + +Try out the converted pipeline +------------------------------ + + + +Now, we are ready to start generation. For improving the generation +process, we also introduce an opportunity to provide a +``negative prompt``. Technically, positive prompt steers the diffusion +toward the output associated with it, while negative prompt steers the +diffusion away from it. The demo app below is created using `Gradio +package `__ + +.. code:: ipython3 + + import gradio as gr + + def _generate(prompt, negative_prompt, audio_length_in_s, + num_inference_steps, _=gr.Progress(track_tqdm=True)): + """Gradio backing function.""" + audio_values = pipe( + prompt, + negative_prompt=negative_prompt, + num_inference_steps=num_inference_steps, + audio_length_in_s=audio_length_in_s + ) + waveform = audio_values[0].squeeze() * 2**15 + return (sampling_rate, waveform.astype(np.int16)) + + demo = gr.Interface( + _generate, + inputs=[ + gr.Textbox(label="Text Prompt"), + gr.Textbox(label="Negative Prompt", placeholder="Example: Low quality"), + gr.Slider( + minimum=1.0, + maximum=15.0, + step=0.25, + value=7, + label="Audio Length (s)", + ), + gr.Slider(label="Inference Steps", step=5, value=150, minimum=50, maximum=250) + ], + outputs=[ + "audio" + ], + examples=[ + ["birds singing in the forest", "Low quality", 7, 150], + ["The sound of a hammer hitting a wooden surface", "", 4, 200], + ], + ) + try: + demo.queue().launch(debug=False) + except Exception: + demo.queue().launch(share=True, debug=False) + + # If you are launching remotely, specify server_name and server_port + # EXAMPLE: `demo.launch(server_name="your server name", server_port="server port in int")` + # To learn more please refer to the Gradio docs: https://gradio.app/docs/ diff --git a/docs/notebooks/270-sound-generation-audioldm2-with-output_files/270-sound-generation-audioldm2.png b/docs/notebooks/270-sound-generation-audioldm2-with-output_files/270-sound-generation-audioldm2.png new file mode 100644 index 00000000000..d865f303a4e --- /dev/null +++ b/docs/notebooks/270-sound-generation-audioldm2-with-output_files/270-sound-generation-audioldm2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722cb1925f39a9c325b6bd8d8ea463794f3028fd9fb44247ef7bf27abb45dde4 +size 600751 diff --git a/docs/notebooks/271-sdxl-turbo-with-output.rst b/docs/notebooks/271-sdxl-turbo-with-output.rst new file mode 100644 index 00000000000..a632cd7181e --- /dev/null +++ b/docs/notebooks/271-sdxl-turbo-with-output.rst @@ -0,0 +1,860 @@ +Single step image generation using SDXL-turbo and OpenVINO +========================================================== + +SDXL-Turbo is a fast generative text-to-image model that can synthesize +photorealistic images from a text prompt in a single network evaluation. +SDXL-Turbo is a distilled version of `SDXL +1.0 `__, +trained for real-time synthesis. SDXL Turbo is based on a novel +distillation technique called Adversarial Diffusion Distillation (ADD), +which enables the model to synthesize image outputs in a single step and +generate real-time text-to-image outputs while maintaining high sampling +fidelity. More details about this distillation approach can be found in +`technical +report `__. +More details about model can be found in `Stability AI blog +post `__. + +Previously, we already discussed how to launch Stable Diffusion XL model +using OpenVINO in the following +`notebook <../248-stable-diffusion-xl>`__, in this tutorial we will +focus on the +`SDXL-turbo `__ version. +Additionally, to improve image decoding speed, we will use `Tiny +Autoencoder `__, which is useful +for real-time previewing of the SDXL generation process. + +We will use a pre-trained model from the `Hugging Face +Diffusers `__ library. To +simplify the user experience, the `Hugging Face Optimum +Intel `__ library is +used to convert the models to OpenVINO™ IR format. + +Prerequisites +------------- + +.. code:: ipython3 + + %pip uninstall -q -y openvino-dev openvino openvino-nightly + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu\ + torch transformers diffusers "git+https://github.com/huggingface/optimum-intel.git" gradio openvino-nightly + +Convert model to OpenVINO format +-------------------------------- + +`sdxl-turbo `__ is +available for downloading via the `HuggingFace +hub `__. We will use optimum-cli +interface for exporting it into OpenVINO Intermediate Representation +(IR) format. + +Optimum CLI interface for converting models supports export to OpenVINO +(supported starting optimum-intel 1.12 version). General command format: + +.. code:: bash + + optimum-cli export openvino --model --task + +where task is task to export the model for, if not specified, the task +will be auto-inferred based on the model. Available tasks depend on the +model, for sdxl should be selected ``stable-diffusion-xl`` + +You can find a mapping between tasks and model classes in Optimum +TaskManager +`documentation `__. + +Additionally, you can specify weights compression ``--fp16`` for the +compression model to FP16 and ``--int8`` for the compression model to +INT8. Please note, that for INT8, it is necessary to install nncf. + +Full list of supported arguments available via ``--help`` For more +details and examples of usage, please check `optimum +documentation `__. + +For Tiny Autoencoder, we will use ``ov.convert_model`` function for +obtaining ``ov.Model`` and save it using ``ov.save_model``. Model +consists of 2 parts that used in pipeline separately: ``vae_encoder`` +for encoding input image in latent space in image-to-image generation +task and ``vae_decoder`` that responsible for decoding diffusion result +back to image format. + +.. code:: ipython3 + + from pathlib import Path + + model_dir = Path("./model") + sdxl_model_id = "stabilityai/sdxl-turbo" + tae_id = "madebyollin/taesdxl" + skip_convert_model = model_dir.exists() + +.. code:: ipython3 + + import torch + import openvino as ov + from diffusers import AutoencoderTiny + import gc + + class VAEEncoder(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, sample): + return self.vae.encode(sample) + + class VAEDecoder(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, latent_sample): + return self.vae.decode(latent_sample) + + def convert_tiny_vae(model_id, output_path): + tiny_vae = AutoencoderTiny.from_pretrained(model_id) + tiny_vae.eval() + vae_encoder = VAEEncoder(tiny_vae) + ov_model = ov.convert_model(vae_encoder, example_input=torch.zeros((1,3,512,512))) + ov.save_model(ov_model, output_path / "vae_encoder/openvino_model.xml") + tiny_vae.save_config(output_path / "vae_encoder") + vae_decoder = VAEDecoder(tiny_vae) + ov_model = ov.convert_model(vae_decoder, example_input=torch.zeros((1,4,64,64))) + ov.save_model(ov_model, output_path / "vae_decoder/openvino_model.xml") + tiny_vae.save_config(output_path / "vae_decoder") + + + if not skip_convert_model: + !optimum-cli export openvino --model $sdxl_model_id --task stable-diffusion-xl $model_dir --fp16 + convert_tiny_vae(tae_id, model_dir) + +Text-to-image generation +------------------------ + +Text-to-image generation lets you create images using text description. +To start generating images, we need to load models first. To load an +OpenVINO model and run an inference with Optimum and OpenVINO Runtime, +you need to replace diffusers ``StableDiffusionXLPipeline`` with Optimum +``OVStableDiffusionXLPipeline``. Pipeline initialization starts with +using ``from_pretrained`` method, where a directory with OpenVINO models +should be passed. Additionally, you can specify an inference device. + +Select inference device for text-to-image generation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: ipython3 + + import ipywidgets as widgets + + core = ov.Core() + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +.. code:: ipython3 + + from optimum.intel.openvino import OVStableDiffusionXLPipeline + text2image_pipe = OVStableDiffusionXLPipeline.from_pretrained(model_dir, device=device.value) + + +.. parsed-literal:: + + INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + + +.. parsed-literal:: + + 2023-12-01 11:21:33.190808: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-01 11:21:33.194252: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. + 2023-12-01 11:21:33.260150: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. + 2023-12-01 11:21:33.261916: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2023-12-01 11:21:33.994990: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + Compiling the vae_decoder to AUTO ... + Compiling the unet to AUTO ... + Compiling the text_encoder_2 to AUTO ... + Compiling the text_encoder to AUTO ... + Compiling the vae_encoder to AUTO ... + + +The pipeline interface is similar to original +``StableDiffusionXLPipeline``. We should provide text prompt. The +default number of steps is 50, while sdxl-turbo required only 1 step. +According to the information provided in model card, model does not use +negative prompt and guidance scale and this parameters should be +disabled using ``guidance_scale = 0`` + +.. code:: ipython3 + + import numpy as np + + prompt = "cute cat" + image = text2image_pipe(prompt, num_inference_steps=1, height=512, width=512, guidance_scale=0.0, generator=np.random.RandomState(987)).images[0] + image.save("cat.png") + image + + +.. parsed-literal:: + + /home/ltalamanova/env_ci/lib/python3.8/site-packages/optimum/intel/openvino/modeling_diffusion.py:565: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + outputs = self.request(inputs, shared_memory=True) + + + +.. parsed-literal:: + + 0%| | 0/1 [00:00= 1 after applying strength multiplication. e.g. in example +below, we will use ``num_inference_steps=2`` and ``stength=0.5``, +finally, we get 0.5 \* 2.0 = 1 step in our pipeline. + +.. code:: ipython3 + + photo_image = image2image_pipe(photo_prompt, image=image, num_inference_steps=2, generator=np.random.RandomState(511), guidance_scale=0.0, strength=0.5).images[0] + photo_image.save("cat_tie.png") + photo_image + + +.. parsed-literal:: + + /home/ltalamanova/env_ci/lib/python3.8/site-packages/optimum/intel/openvino/modeling_diffusion.py:636: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + outputs = self.request(inputs, shared_memory=True) + + + +.. parsed-literal:: + + 0%| | 0/1 [00:00`__ enables +post-training quantization by adding quantization layers into model +graph and then using a subset of the training dataset to initialize the +parameters of these additional quantization layers. Quantized operations +are executed in ``INT8`` instead of ``FP32``/``FP16`` making model +inference faster. + +According to ``SDXL-Turbo Model`` structure, the UNet model takes up +significant portion of the overall pipeline execution time. Now we will +show you how to optimize the UNet part using +`NNCF `__ to reduce +computation cost and speed up the pipeline. Quantizing the rest of the +SDXL pipeline does not significantly improve inference performance but +can lead to a substantial degradation of accuracy. + +The optimization process contains the following steps: + +1. Create a calibration dataset for quantization. +2. Run ``nncf.quantize()`` to obtain quantized model. +3. Save the ``INT8`` model using ``openvino.save_model()`` function. + +Please select below whether you would like to run quantization to +improve model inference speed. + +.. code:: ipython3 + + to_quantize = widgets.Checkbox( + value=True, + description='Quantization', + disabled=False, + ) + + to_quantize + + + + +.. parsed-literal:: + + Checkbox(value=True, description='Quantization') + + + +.. code:: ipython3 + + import sys + sys.path.append("../utils") + + int8_pipe = None + + if to_quantize.value and "GPU" in device.value: + to_quantize.value = False + + %load_ext skip_kernel_extension + +Prepare calibration dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We use a portion of +`laion/laion2B-en `__ +dataset from Hugging Face as calibration data. To collect intermediate +model inputs for calibration we should customize ``CompiledModel``. + +.. code:: ipython3 + + UNET_INT8_OV_PATH = model_dir / "optimized_unet" / "openvino_model.xml" + + def disable_progress_bar(pipeline, disable=True): + if not hasattr(pipeline, "_progress_bar_config"): + pipeline._progress_bar_config = {'disable': disable} + else: + pipeline._progress_bar_config['disable'] = disable + +.. code:: ipython3 + + %%skip not $to_quantize.value + + import datasets + import numpy as np + from tqdm.notebook import tqdm + from transformers import set_seed + from typing import Any, Dict, List + + set_seed(1) + + class CompiledModelDecorator(ov.CompiledModel): + def __init__(self, compiled_model: ov.CompiledModel, data_cache: List[Any] = None): + super().__init__(compiled_model) + self.data_cache = data_cache if data_cache else [] + + def __call__(self, *args, **kwargs): + self.data_cache.append(*args) + return super().__call__(*args, **kwargs) + + def collect_calibration_data(pipe, subset_size: int) -> List[Dict]: + original_unet = pipe.unet.request + pipe.unet.request = CompiledModelDecorator(original_unet) + + dataset = datasets.load_dataset("laion/laion2B-en", split="train", streaming=True).shuffle(seed=42) + disable_progress_bar(pipe) + + # Run inference for data collection + pbar = tqdm(total=subset_size) + diff = 0 + for batch in dataset: + prompt = batch["TEXT"] + if len(prompt) > pipe.tokenizer.model_max_length: + continue + _ = pipe( + prompt, + num_inference_steps=1, + height=512, + width=512, + guidance_scale=0.0, + generator=np.random.RandomState(987) + ) + collected_subset_size = len(pipe.unet.request.data_cache) + if collected_subset_size >= subset_size: + pbar.update(subset_size - pbar.n) + break + pbar.update(collected_subset_size - diff) + diff = collected_subset_size + + calibration_dataset = pipe.unet.request.data_cache + disable_progress_bar(pipe, disable=False) + pipe.unet.request = original_unet + return calibration_dataset + +.. code:: ipython3 + + %%skip not $to_quantize.value + + if not UNET_INT8_OV_PATH.exists(): + text2image_pipe = OVStableDiffusionXLPipeline.from_pretrained(model_dir, device=device.value) + unet_calibration_data = collect_calibration_data(text2image_pipe, subset_size=200) + + +.. parsed-literal:: + + Compiling the vae_decoder to AUTO ... + Compiling the unet to AUTO ... + Compiling the text_encoder_2 to AUTO ... + Compiling the vae_encoder to AUTO ... + Compiling the text_encoder to AUTO ... + + + +.. parsed-literal:: + + Resolving data files: 0%| | 0/128 [00:00:17: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + /home/ltalamanova/env_ci/lib/python3.8/site-packages/optimum/intel/openvino/modeling_diffusion.py:615: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + outputs = self.request(inputs, shared_memory=True) + + +Run quantization +~~~~~~~~~~~~~~~~ + +Create a quantized model from the pre-trained converted OpenVINO model. +Quantization of the first and last ``Convolution`` layers impacts the +generation results. We recommend using ``IgnoredScope`` to keep accuracy +sensitive ``Convolution`` layers in FP16 precision. + + **NOTE**: Quantization is time and memory consuming operation. + Running quantization code below may take some time. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + import nncf + from nncf.scopes import IgnoredScope + + UNET_OV_PATH = model_dir / "unet" / "openvino_model.xml" + if not UNET_INT8_OV_PATH.exists(): + unet = core.read_model(UNET_OV_PATH) + quantized_unet = nncf.quantize( + model=unet, + model_type=nncf.ModelType.TRANSFORMER, + calibration_dataset=nncf.Dataset(unet_calibration_data), + ignored_scope=IgnoredScope( + names=[ + "__module.model.conv_in/aten::_convolution/Convolution", + "__module.model.up_blocks.2.resnets.2.conv_shortcut/aten::_convolution/Convolution", + "__module.model.conv_out/aten::_convolution/Convolution" + ], + ), + ) + ov.save_model(quantized_unet, UNET_INT8_OV_PATH) + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + +.. parsed-literal:: + + INFO:nncf:3 ignored nodes were found by name in the NNCFGraph + INFO:nncf:420 ignored nodes were found by name in the NNCFGraph + INFO:nncf:Not adding activation input quantizer for operation: 5 __module.model.conv_in/aten::_convolution/Convolution + 13 __module.model.conv_in/aten::_convolution/Add_87 + + INFO:nncf:Not adding activation input quantizer for operation: 460 __module.model.up_blocks.2.resnets.2.conv_shortcut/aten::_convolution/Convolution + 899 __module.model.up_blocks.2.resnets.2.conv_shortcut/aten::_convolution/Add_16859 + + INFO:nncf:Not adding activation input quantizer for operation: 3911 __module.model.conv_out/aten::_convolution/Convolution + 4032 __module.model.conv_out/aten::_convolution/Add_16873 + + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + +

+
+
+
+
+.. raw:: html
+
+    
+    
+ + + +Let us check predictions with the quantized UNet using the same input +data. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + from IPython.display import display + + int8_text2image_pipe = OVStableDiffusionXLPipeline.from_pretrained(model_dir, device=device.value, compile=False) + int8_text2image_pipe.unet.model = core.read_model(UNET_INT8_OV_PATH) + int8_text2image_pipe.unet.request = None + + prompt = "cute cat" + image = int8_text2image_pipe(prompt, num_inference_steps=1, height=512, width=512, guidance_scale=0.0, generator=np.random.RandomState(987)).images[0] + display(image) + + +.. parsed-literal:: + + Compiling the text_encoder to AUTO ... + /home/ltalamanova/env_ci/lib/python3.8/site-packages/optimum/intel/openvino/modeling_diffusion.py:565: FutureWarning: `shared_memory` is deprecated and will be removed in 2024.0. Value of `shared_memory` is going to override `share_inputs` value. Please use only `share_inputs` explicitly. + outputs = self.request(inputs, shared_memory=True) + Compiling the text_encoder_2 to AUTO ... + + + +.. parsed-literal:: + + 0%| | 0/1 [00:00 +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/271-sdxl-turbo-with-output_files/ + +

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/271-sdxl-turbo-with-output_files/


../
+271-sdxl-turbo-with-output_11_3.jpg                07-Dec-2023 00:49               28118
+271-sdxl-turbo-with-output_11_3.png                07-Dec-2023 00:49              376542
+271-sdxl-turbo-with-output_17_2.jpg                07-Dec-2023 00:49               28230
+271-sdxl-turbo-with-output_17_2.png                07-Dec-2023 00:49              398923
+271-sdxl-turbo-with-output_29_3.jpg                07-Dec-2023 00:49               27869
+271-sdxl-turbo-with-output_29_3.png                07-Dec-2023 00:49              375737
+271-sdxl-turbo-with-output_30_1.jpg                07-Dec-2023 00:49               27505
+271-sdxl-turbo-with-output_30_1.png                07-Dec-2023 00:49              386457
+

+ diff --git a/docs/notebooks/272-paint-by-example-with-output.rst b/docs/notebooks/272-paint-by-example-with-output.rst new file mode 100644 index 00000000000..abdc8266dad --- /dev/null +++ b/docs/notebooks/272-paint-by-example-with-output.rst @@ -0,0 +1,1104 @@ +Paint By Example: Exemplar-based Image Editing with Diffusion Models +==================================================================== + +Stable Diffusion in Diffusers library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To work with Stable Diffusion, we will use the Hugging Face +`Diffusers `__ library. To +experiment with in-painting we can use Diffusers which exposes the +`StableDiffusionInpaintPipeline `__ +similar to the `other Diffusers +pipelines `__. +The code below demonstrates how to create +``StableDiffusionInpaintPipeline`` using +``stable-diffusion-2-inpainting``. To create the drawing tool we will +install Gradio for handling user interaction. + +This is the overall flow of the application: |Flow Diagram| + +This is the detailed flowchart for the pipeline: |pipeline-flowchart| + +.. |Flow Diagram| image:: https://user-images.githubusercontent.com/103226580/236954918-f364b227-293c-4f78-a9bf-9dcebcb1034a.png +.. |pipeline-flowchart| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/103226580/cde2d5c4-2540-4a45-ad9c-339f7a69459d + +.. code:: ipython3 + + %pip install -q "gradio == 3.50.2" + %pip install -q "diffusers>=-1.14.0" "openvino>=2023.2.0" "transformers >= 4.25.1" + + +.. parsed-literal:: + + Collecting gradio==3.50.2 + Downloading gradio-3.50.2-py3-none-any.whl (20.3 MB) + 0.0/20.3 MB ? eta -:--:-- + 0.4/20.3 MB 8.1 MB/s eta 0:00:03 + - 1.0/20.3 MB 10.2 MB/s eta 0:00:02 + --- 1.6/20.3 MB 11.0 MB/s eta 0:00:02 + --- 1.9/20.3 MB 10.1 MB/s eta 0:00:02 + ---- 2.3/20.3 MB 10.5 MB/s eta 0:00:02 + ----- 2.6/20.3 MB 9.8 MB/s eta 0:00:02 + ------ 3.1/20.3 MB 9.8 MB/s eta 0:00:02 + ------ 3.5/20.3 MB 9.7 MB/s eta 0:00:02 + ------- 4.0/20.3 MB 9.8 MB/s eta 0:00:02 + -------- 4.4/20.3 MB 9.7 MB/s eta 0:00:02 + --------- 4.9/20.3 MB 9.8 MB/s eta 0:00:02 + ---------- 5.4/20.3 MB 9.8 MB/s eta 0:00:02 + ----------- 5.9/20.3 MB 9.9 MB/s eta 0:00:02 + ------------ 6.4/20.3 MB 10.2 MB/s eta 0:00:02 + ------------- 6.9/20.3 MB 10.2 MB/s eta 0:00:02 + -------------- 7.4/20.3 MB 10.3 MB/s eta 0:00:02 + --------------- 7.9/20.3 MB 10.3 MB/s eta 0:00:02 + ---------------- 8.3/20.3 MB 10.2 MB/s eta 0:00:02 + ----------------- 8.8/20.3 MB 10.3 MB/s eta 0:00:02 + ------------------ 9.4/20.3 MB 10.4 MB/s eta 0:00:02 + ------------------- 9.9/20.3 MB 10.4 MB/s eta 0:00:01 + ------------------- 10.3/20.3 MB 10.2 MB/s eta 0:00:01 + -------------------- 10.8/20.3 MB 10.2 MB/s eta 0:00:01 + --------------------- 11.1/20.3 MB 10.2 MB/s eta 0:00:01 + ---------------------- 11.6/20.3 MB 10.1 MB/s eta 0:00:01 + ----------------------- 12.1/20.3 MB 10.2 MB/s eta 0:00:01 + ------------------------ 12.6/20.3 MB 10.4 MB/s eta 0:00:01 + ------------------------- 13.0/20.3 MB 10.4 MB/s eta 0:00:01 + ------------------------- 13.5/20.3 MB 10.4 MB/s eta 0:00:01 + -------------------------- 14.0/20.3 MB 10.4 MB/s eta 0:00:01 + --------------------------- 14.5/20.3 MB 10.6 MB/s eta 0:00:01 + ---------------------------- 15.0/20.3 MB 10.6 MB/s eta 0:00:01 + ----------------------------- 15.5/20.3 MB 10.6 MB/s eta 0:00:01 + ------------------------------ 16.0/20.3 MB 10.6 MB/s eta 0:00:01 + ------------------------------- 16.5/20.3 MB 10.6 MB/s eta 0:00:01 + -------------------------------- 17.1/20.3 MB 10.6 MB/s eta 0:00:01 + --------------------------------- 17.5/20.3 MB 10.7 MB/s eta 0:00:01 + ---------------------------------- 18.0/20.3 MB 10.7 MB/s eta 0:00:01 + ----------------------------------- 18.5/20.3 MB 10.7 MB/s eta 0:00:01 + ------------------------------------ 18.9/20.3 MB 10.6 MB/s eta 0:00:01 + ------------------------------------- 19.4/20.3 MB 10.6 MB/s eta 0:00:01 + -------------------------------------- 19.8/20.3 MB 10.4 MB/s eta 0:00:01 + -------------------------------------- 20.3/20.3 MB 10.6 MB/s eta 0:00:01 + -------------------------------------- 20.3/20.3 MB 10.6 MB/s eta 0:00:01 + -------------------------------------- 20.3/20.3 MB 10.6 MB/s eta 0:00:01 + -------------------------------------- 20.3/20.3 MB 10.6 MB/s eta 0:00:01 + -------------------------------------- 20.3/20.3 MB 10.6 MB/s eta 0:00:01 + ---------------------------------------- 20.3/20.3 MB 8.5 MB/s eta 0:00:00 + Requirement already satisfied: aiofiles<24.0,>=22.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (22.1.0) + Requirement already satisfied: altair<6.0,>=4.2.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (4.2.2) + Requirement already satisfied: fastapi in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.95.1) + Requirement already satisfied: ffmpy in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.3.0) + Collecting gradio-client==0.6.1 (from gradio==3.50.2) + Downloading gradio_client-0.6.1-py3-none-any.whl (299 kB) + 0.0/299.2 kB ? eta -:--:-- + -------------------------------------- 299.2/299.2 kB 6.3 MB/s eta 0:00:00 + Requirement already satisfied: httpx in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.24.0) + Requirement already satisfied: huggingface-hub>=0.14.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.14.1) + Collecting importlib-resources<7.0,>=1.3 (from gradio==3.50.2) + Downloading importlib_resources-6.1.1-py3-none-any.whl (33 kB) + Requirement already satisfied: jinja2<4.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (3.1.2) + Requirement already satisfied: markupsafe~=2.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (2.1.2) + Requirement already satisfied: matplotlib~=3.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (3.5.2) + Requirement already satisfied: numpy~=1.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (1.23.4) + Requirement already satisfied: orjson~=3.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (3.8.11) + Requirement already satisfied: packaging in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (23.1) + Requirement already satisfied: pandas<3.0,>=1.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (1.3.5) + Requirement already satisfied: pillow<11.0,>=8.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (9.5.0) + Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (1.10.7) + Requirement already satisfied: pydub in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.25.1) + Requirement already satisfied: python-multipart in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.0.6) + Requirement already satisfied: pyyaml<7.0,>=5.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (6.0) + Requirement already satisfied: requests~=2.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (2.29.0) + Requirement already satisfied: semantic-version~=2.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (2.10.0) + Requirement already satisfied: typing-extensions~=4.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (4.5.0) + Requirement already satisfied: uvicorn>=0.14.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (0.22.0) + Requirement already satisfied: websockets<12.0,>=10.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio==3.50.2) (11.0.2) + Requirement already satisfied: fsspec in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from gradio-client==0.6.1->gradio==3.50.2) (2023.4.0) + Requirement already satisfied: entrypoints in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from altair<6.0,>=4.2.0->gradio==3.50.2) (0.4) + Requirement already satisfied: jsonschema>=3.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from altair<6.0,>=4.2.0->gradio==3.50.2) (4.17.3) + Requirement already satisfied: toolz in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from altair<6.0,>=4.2.0->gradio==3.50.2) (0.12.0) + Requirement already satisfied: filelock in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from huggingface-hub>=0.14.0->gradio==3.50.2) (3.12.0) + Requirement already satisfied: tqdm>=4.42.1 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from huggingface-hub>=0.14.0->gradio==3.50.2) (4.65.0) + Requirement already satisfied: cycler>=0.10 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from matplotlib~=3.0->gradio==3.50.2) (0.11.0) + Requirement already satisfied: fonttools>=4.22.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from matplotlib~=3.0->gradio==3.50.2) (4.39.3) + Requirement already satisfied: kiwisolver>=1.0.1 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from matplotlib~=3.0->gradio==3.50.2) (1.4.4) + Requirement already satisfied: pyparsing>=2.2.1 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from matplotlib~=3.0->gradio==3.50.2) (2.4.7) + Requirement already satisfied: python-dateutil>=2.7 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from matplotlib~=3.0->gradio==3.50.2) (2.8.2) + Requirement already satisfied: pytz>=2017.3 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from pandas<3.0,>=1.0->gradio==3.50.2) (2023.3) + Requirement already satisfied: charset-normalizer<4,>=2 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from requests~=2.0->gradio==3.50.2) (3.1.0) + Requirement already satisfied: idna<4,>=2.5 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from requests~=2.0->gradio==3.50.2) (3.4) + Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from requests~=2.0->gradio==3.50.2) (1.26.15) + Requirement already satisfied: certifi>=2017.4.17 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from requests~=2.0->gradio==3.50.2) (2022.12.7) + Requirement already satisfied: click>=7.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from uvicorn>=0.14.0->gradio==3.50.2) (8.1.3) + Requirement already satisfied: h11>=0.8 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from uvicorn>=0.14.0->gradio==3.50.2) (0.14.0) + Requirement already satisfied: starlette<0.27.0,>=0.26.1 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from fastapi->gradio==3.50.2) (0.26.1) + Requirement already satisfied: httpcore<0.18.0,>=0.15.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from httpx->gradio==3.50.2) (0.17.0) + Requirement already satisfied: sniffio in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from httpx->gradio==3.50.2) (1.3.0) + Requirement already satisfied: colorama in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from click>=7.0->uvicorn>=0.14.0->gradio==3.50.2) (0.4.6) + Requirement already satisfied: anyio<5.0,>=3.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from httpcore<0.18.0,>=0.15.0->httpx->gradio==3.50.2) (3.6.2) + Requirement already satisfied: attrs>=17.4.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.50.2) (23.1.0) + Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.50.2) (0.19.3) + Requirement already satisfied: six>=1.5 in c:\hackathon\openvino_notebooks\venv310\lib\site-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio==3.50.2) (1.16.0) + Installing collected packages: importlib-resources, gradio-client, gradio + Attempting uninstall: gradio-client + Found existing installation: gradio_client 0.1.4 + Uninstalling gradio_client-0.1.4: + Successfully uninstalled gradio_client-0.1.4 + Attempting uninstall: gradio + Found existing installation: gradio 3.28.1 + Uninstalling gradio-3.28.1: + Successfully uninstalled gradio-3.28.1 + Successfully installed gradio-3.50.2 gradio-client-0.6.1 importlib-resources-6.1.1 + Note: you may need to restart the kernel to use updated packages. + + +.. parsed-literal:: + + + [notice] A new release of pip is available: 23.1 -> 23.3.1 + [notice] To update, run: python.exe -m pip install --upgrade pip + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + + +.. parsed-literal:: + + + [notice] A new release of pip is available: 23.1 -> 23.3.1 + [notice] To update, run: python.exe -m pip install --upgrade pip + + +Download the model from `HuggingFace +Paint-by-Example `__. +This might take several minutes because it is over 5GB + +.. code:: ipython3 + + from diffusers import DPMSolverMultistepScheduler, DiffusionPipeline + + pipeline = DiffusionPipeline.from_pretrained("Fantasy-Studio/Paint-By-Example") + + scheduler_inpaint = DPMSolverMultistepScheduler.from_config(pipeline.scheduler.config) + + +.. parsed-literal:: + + Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: + ``` + pip install accelerate + ``` + . + You are using a model of type clip_vision_model to instantiate a model of type clip. This is not supported for all configurations of models and can yield errors. + + +.. code:: ipython3 + + import gc + + extractor = pipeline.feature_extractor + image_encoder = pipeline.image_encoder + image_encoder.eval() + unet_inpaint = pipeline.unet + unet_inpaint.eval() + vae_inpaint = pipeline.vae + vae_inpaint.eval() + + del pipeline + gc.collect(); + +Download default images +~~~~~~~~~~~~~~~~~~~~~~~ + +Download default images. + +.. code:: ipython3 + + # Fetch `notebook_utils` module + import urllib.request + urllib.request.urlretrieve( + url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', + filename='notebook_utils.py' + ) + + from notebook_utils import download_file + + download_file("https://github-production-user-asset-6210df.s3.amazonaws.com/103226580/286377210-edc98e97-0e43-4796-b771-dacd074c39ea.png", "0.png", "data/image") + + download_file("https://github-production-user-asset-6210df.s3.amazonaws.com/103226580/286377233-b2c2d902-d379-415a-8183-5bdd37c52429.png", "1.png", "data/image") + + download_file("https://github-production-user-asset-6210df.s3.amazonaws.com/103226580/286377248-da1db61e-3521-4cdb-85c8-1386d360ce22.png", "2.png", "data/image") + + download_file("https://github-production-user-asset-6210df.s3.amazonaws.com/103226580/286377279-fa496f17-e850-4351-87c5-2552dfbc4633.jpg", "bird.jpg", "data/reference") + + download_file("https://github-production-user-asset-6210df.s3.amazonaws.com/103226580/286377298-06a25ff2-84d8-4d46-95cd-8c25efa690d8.jpg", "car.jpg", "data/reference") + + download_file("https://github-production-user-asset-6210df.s3.amazonaws.com/103226580/286377318-8841a801-1933-4523-a433-7d2fb64c47e6.jpg", "dog.jpg", "data/reference") + + + + +.. parsed-literal:: + + data\image\0.png: 0%| | 0.00/453k [00:00`__ + +.. code:: ipython3 + + from pathlib import Path + import torch + import numpy as np + import openvino as ov + + model_dir = Path("model") + model_dir.mkdir(exist_ok=True) + sd2_inpainting_model_dir = Path("model/paint_by_example") + sd2_inpainting_model_dir.mkdir(exist_ok=True) + +Functions to convert to OpenVINO IR format + +.. code:: ipython3 + + def cleanup_torchscript_cache(): + """ + Helper for removing cached model representation + """ + torch._C._jit_clear_class_registry() + torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() + torch.jit._state._clear_class_state() + + + def convert_image_encoder(image_encoder: torch.nn.Module, ir_path:Path): + """ + Convert Image Encoder model to IR. + Function accepts pipeline, prepares example inputs for conversion + Parameters: + image_encoder (torch.nn.Module): image encoder PyTorch model + ir_path (Path): File for storing model + Returns: + None + """ + class ImageEncoderWrapper(torch.nn.Module): + def __init__(self, image_encoder): + super().__init__() + self.image_encoder = image_encoder + + def forward(self, image): + image_embeddings, negative_prompt_embeds = self.image_encoder(image, return_uncond_vector=True) + return image_embeddings, negative_prompt_embeds + + if not ir_path.exists(): + image_encoder = ImageEncoderWrapper(image_encoder) + image_encoder.eval() + input_ids = torch.randn((1,3,224,224)) + # switch model to inference mode + + # disable gradients calculation for reducing memory consumption + with torch.no_grad(): + ov_model = ov.convert_model( + image_encoder, + example_input=input_ids, + input=([1,3,224,224],) + ) + ov.save_model(ov_model, ir_path) + del ov_model + cleanup_torchscript_cache() + print('Image Encoder successfully converted to IR') + + + def convert_unet(unet:torch.nn.Module, ir_path:Path, num_channels:int = 4, width:int = 64, height:int = 64): + """ + Convert Unet model to IR format. + Function accepts pipeline, prepares example inputs for conversion + Parameters: + unet (torch.nn.Module): UNet PyTorch model + ir_path (Path): File for storing model + num_channels (int, optional, 4): number of input channels + width (int, optional, 64): input width + height (int, optional, 64): input height + Returns: + None + """ + dtype_mapping = { + torch.float32: ov.Type.f32, + torch.float64: ov.Type.f64 + } + if not ir_path.exists(): + # prepare inputs + encoder_hidden_state = torch.ones((2, 1, 768)) + latents_shape = (2, num_channels, width, height) + latents = torch.randn(latents_shape) + t = torch.from_numpy(np.array(1, dtype=np.float32)) + unet.eval() + dummy_inputs = (latents, t, encoder_hidden_state) + input_info = [] + for input_tensor in dummy_inputs: + shape = ov.PartialShape(tuple(input_tensor.shape)) + element_type = dtype_mapping[input_tensor.dtype] + input_info.append((shape, element_type)) + + with torch.no_grad(): + ov_model = ov.convert_model( + unet, + example_input=dummy_inputs, + input=input_info + ) + ov.save_model(ov_model, ir_path) + del ov_model + cleanup_torchscript_cache() + print('U-Net successfully converted to IR') + + + def convert_vae_encoder(vae: torch.nn.Module, ir_path: Path, width:int = 512, height:int = 512): + """ + Convert VAE model to IR format. + Function accepts VAE model, creates wrapper class for export only necessary for inference part, + prepares example inputs for conversion, + Parameters: + vae (torch.nn.Module): VAE PyTorch model + ir_path (Path): File for storing model + width (int, optional, 512): input width + height (int, optional, 512): input height + Returns: + None + """ + class VAEEncoderWrapper(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, image): + latents = self.vae.encode(image).latent_dist.sample() + return latents + + if not ir_path.exists(): + vae_encoder = VAEEncoderWrapper(vae) + vae_encoder.eval() + image = torch.zeros((1, 3, width, height)) + with torch.no_grad(): + ov_model = ov.convert_model(vae_encoder, example_input=image, input=([1,3, width, height],)) + ov.save_model(ov_model, ir_path) + del ov_model + cleanup_torchscript_cache() + print('VAE encoder successfully converted to IR') + + + def convert_vae_decoder(vae: torch.nn.Module, ir_path: Path, width:int = 64, height:int = 64): + """ + Convert VAE decoder model to IR format. + Function accepts VAE model, creates wrapper class for export only necessary for inference part, + prepares example inputs for conversion, + Parameters: + vae (torch.nn.Module): VAE model + ir_path (Path): File for storing model + width (int, optional, 64): input width + height (int, optional, 64): input height + Returns: + None + """ + class VAEDecoderWrapper(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, latents): + latents = 1 / 0.18215 * latents + return self.vae.decode(latents) + + if not ir_path.exists(): + vae_decoder = VAEDecoderWrapper(vae) + latents = torch.zeros((1, 4, width, height)) + + vae_decoder.eval() + with torch.no_grad(): + ov_model = ov.convert_model(vae_decoder, example_input=latents, input=([1, 4, width, height],)) + ov.save_model(ov_model, ir_path) + del ov_model + cleanup_torchscript_cache() + print('VAE decoder successfully converted to ') + +Do the conversion of the in-painting model: + +.. code:: ipython3 + + IMAGE_ENCODER_OV_PATH_INPAINT = sd2_inpainting_model_dir / "image_encoder.xml" + + if not IMAGE_ENCODER_OV_PATH_INPAINT.exists(): + convert_image_encoder(image_encoder, IMAGE_ENCODER_OV_PATH_INPAINT) + else: + print(f"Image encoder will be loaded from {IMAGE_ENCODER_OV_PATH_INPAINT}") + + del image_encoder + gc.collect(); + + +.. parsed-literal:: + + Image encoder will be loaded from model\paint_by_example\image_encoder.xml + + +Do the conversion of the Unet model + +.. code:: ipython3 + + UNET_OV_PATH_INPAINT = sd2_inpainting_model_dir / 'unet.xml' + if not UNET_OV_PATH_INPAINT.exists(): + convert_unet(unet_inpaint, UNET_OV_PATH_INPAINT, num_channels=9, width=64, height=64) + del unet_inpaint + gc.collect() + else: + del unet_inpaint + print(f"U-Net will be loaded from {UNET_OV_PATH_INPAINT}") + gc.collect(); + + +.. parsed-literal:: + + U-Net will be loaded from model\paint_by_example\unet.xml + + +Do the conversion of the VAE Encoder model + +.. code:: ipython3 + + VAE_ENCODER_OV_PATH_INPAINT = sd2_inpainting_model_dir / 'vae_encoder.xml' + + if not VAE_ENCODER_OV_PATH_INPAINT.exists(): + convert_vae_encoder(vae_inpaint, VAE_ENCODER_OV_PATH_INPAINT, 512, 512) + else: + print(f"VAE encoder will be loaded from {VAE_ENCODER_OV_PATH_INPAINT}") + + VAE_DECODER_OV_PATH_INPAINT = sd2_inpainting_model_dir / 'vae_decoder.xml' + if not VAE_DECODER_OV_PATH_INPAINT.exists(): + convert_vae_decoder(vae_inpaint, VAE_DECODER_OV_PATH_INPAINT, 64, 64) + else: + print(f"VAE decoder will be loaded from {VAE_DECODER_OV_PATH_INPAINT}") + + del vae_inpaint + gc.collect(); + + +.. parsed-literal:: + + VAE encoder will be loaded from model\paint_by_example\vae_encoder.xml + VAE decoder will be loaded from model\paint_by_example\vae_decoder.xml + + +Prepare Inference pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Function to prepare the mask and masked image. + +Adapted from `236 Stable Diffusion v2 Infinite Zoom +notebook <236-stable-diffusion-v2-with-output.html>`__ + +The main difference is that instead of encoding a text prompt it will +now encode an image as the prompt. + +.. code:: ipython3 + + import inspect + from typing import Optional, Union, Dict + + import PIL + import cv2 + + from transformers import CLIPImageProcessor + from diffusers.pipelines.pipeline_utils import DiffusionPipeline + from diffusers.schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler + from openvino.runtime import Model + + + def prepare_mask_and_masked_image(image:PIL.Image.Image, mask:PIL.Image.Image): + """ + Prepares a pair (image, mask) to be consumed by the Stable Diffusion pipeline. This means that those inputs will be + converted to ``np.array`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the + ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``np.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``np.float32`` too. + + Args: + image (Union[np.array, PIL.Image]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array``. + + Returns: + tuple[np.array]: The pair (mask, masked_image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = image.astype(np.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask = 1 - mask + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + masked_image = image * mask + + return mask, masked_image + +Class for the pipeline which will connect all the models together: VAE +decode –> image encode –> tokenizer –> Unet –> VAE model –> scheduler + +.. code:: ipython3 + + class OVStableDiffusionInpaintingPipeline(DiffusionPipeline): + def __init__( + self, + vae_decoder: Model, + image_encoder: Model, + image_processor: CLIPImageProcessor, + unet: Model, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + vae_encoder: Model = None, + ): + """ + Pipeline for text-to-image generation using Stable Diffusion. + Parameters: + vae_decoder (Model): + Variational Auto-Encoder (VAE) Model to decode images to and from latent representations. + image_encoder (Model): + https://huggingface.co/Fantasy-Studio/Paint-by-Example/blob/main/image_encoder/config.json + tokenizer (CLIPTokenizer): + Tokenizer of class CLIPTokenizer(https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet (Model): Conditional U-Net architecture to denoise the encoded image latents. + vae_encoder (Model): + Variational Auto-Encoder (VAE) Model to encode images to latent representation. + scheduler (SchedulerMixin): + A scheduler to be used in combination with unet to denoise the encoded image latents. Can be one of + DDIMScheduler, LMSDiscreteScheduler, or PNDMScheduler. + """ + super().__init__() + self.scheduler = scheduler + self.vae_decoder = vae_decoder + self.vae_encoder = vae_encoder + self.image_encoder = image_encoder + self.unet = unet + self._unet_output = unet.output(0) + self._vae_d_output = vae_decoder.output(0) + self._vae_e_output = vae_encoder.output(0) if vae_encoder is not None else None + self.height = self.unet.input(0).shape[2] * 8 + self.width = self.unet.input(0).shape[3] * 8 + self.image_processor = image_processor + + def prepare_mask_latents( + self, + mask, + masked_image, + height=512, + width=512, + do_classifier_free_guidance=True, + ): + """ + Prepare mask as Unet nput and encode input masked image to latent space using vae encoder + + Parameters: + mask (np.array): input mask array + masked_image (np.array): masked input image tensor + heigh (int, *optional*, 512): generated image height + width (int, *optional*, 512): generated image width + do_classifier_free_guidance (bool, *optional*, True): whether to use classifier free guidance or not + Returns: + mask (np.array): resized mask tensor + masked_image_latents (np.array): masked image encoded into latent space using VAE + """ + mask = torch.nn.functional.interpolate(torch.from_numpy(mask), size=(height // 8, width // 8)) + mask = mask.numpy() + + # encode the mask image into latents space so we can concatenate it to the latents + masked_image_latents = self.vae_encoder(masked_image)[self._vae_e_output] + masked_image_latents = 0.18215 * masked_image_latents + + mask = np.concatenate([mask] * 2) if do_classifier_free_guidance else mask + masked_image_latents = ( + np.concatenate([masked_image_latents] * 2) + if do_classifier_free_guidance + else masked_image_latents + ) + return mask, masked_image_latents + + def __call__( + self, + image: PIL.Image.Image, + mask_image: PIL.Image.Image, + reference_image: PIL.Image.Image, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + eta: Optional[float] = 0, + output_type: Optional[str] = "pil", + seed: Optional[int] = None, + ): + """ + Function invoked when calling the pipeline for generation. + Parameters: + image (PIL.Image.Image): + Source image for inpainting. + mask_image (PIL.Image.Image): + Mask area for inpainting + reference_image (PIL.Image.Image): + Reference image to inpaint in mask area + num_inference_steps (int, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (float, *optional*, defaults to 7.5): + Guidance scale as defined in Classifier-Free Diffusion Guidance(https://arxiv.org/abs/2207.12598). + guidance_scale is defined as `w` of equation 2. + Higher guidance scale encourages to generate images that are closely linked to the text prompt, + usually at the expense of lower image quality. + eta (float, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [DDIMScheduler], will be ignored for others. + output_type (`str`, *optional*, defaults to "pil"): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): PIL.Image.Image or np.array. + seed (int, *optional*, None): + Seed for random generator state initialization. + Returns: + Dictionary with keys: + sample - the last generated image PIL.Image.Image or np.array + """ + if seed is not None: + np.random.seed(seed) + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # get reference image embeddings + image_embeddings = self._encode_image(reference_image, do_classifier_free_guidance=do_classifier_free_guidance) + + # prepare mask + mask, masked_image = prepare_mask_and_masked_image(image, mask_image) + # set timesteps + accepts_offset = "offset" in set( + inspect.signature(self.scheduler.set_timesteps).parameters.keys() + ) + extra_set_kwargs = {} + if accepts_offset: + extra_set_kwargs["offset"] = 1 + + self.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, 1) + latent_timestep = timesteps[:1] + + # get the initial random noise unless the user supplied it + latents, meta = self.prepare_latents(None, latent_timestep) + mask, masked_image_latents = self.prepare_mask_latents( + mask, + masked_image, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set( + inspect.signature(self.scheduler.step).parameters.keys() + ) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + for t in self.progress_bar(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = ( + np.concatenate([latents] * 2) + if do_classifier_free_guidance + else latents + ) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + latent_model_input = np.concatenate( + [latent_model_input, masked_image_latents, mask], axis=1 + ) + # predict the noise residual + noise_pred = self.unet( + [latent_model_input, np.array(t, dtype=np.float32), image_embeddings] + )[self._unet_output] + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[0], noise_pred[1] + noise_pred = noise_pred_uncond + guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + torch.from_numpy(noise_pred), + t, + torch.from_numpy(latents), + **extra_step_kwargs, + )["prev_sample"].numpy() + # scale and decode the image latents with vae + image = self.vae_decoder(latents)[self._vae_d_output] + + image = self.postprocess_image(image, meta, output_type) + return {"sample": image} + + def _encode_image(self, image:PIL.Image.Image, do_classifier_free_guidance:bool = True): + """ + Encodes the image into image encoder hidden states. + + Parameters: + image (PIL.Image.Image): base image to encode + do_classifier_free_guidance (bool): whether to use classifier free guidance or not + Returns: + image_embeddings (np.ndarray): image encoder hidden states + """ + processed_image = self.image_processor(image) + processed_image = processed_image['pixel_values'][0] + processed_image = np.expand_dims(processed_image, axis=0) + + output = self.image_encoder(processed_image) + image_embeddings = output[self.image_encoder.output(0)] + negative_embeddings = output[self.image_encoder.output(1)] + + image_embeddings = np.concatenate([negative_embeddings, image_embeddings]) + + return image_embeddings + + def prepare_latents(self, image:PIL.Image.Image = None, latent_timestep:torch.Tensor = None): + """ + Function for getting initial latents for starting generation + + Parameters: + image (PIL.Image.Image, *optional*, None): + Input image for generation, if not provided randon noise will be used as starting point + latent_timestep (torch.Tensor, *optional*, None): + Predicted by scheduler initial step for image generation, required for latent image mixing with nosie + Returns: + latents (np.ndarray): + Image encoded in latent space + """ + latents_shape = (1, 4, self.height // 8, self.width // 8) + noise = np.random.randn(*latents_shape).astype(np.float32) + if image is None: + # if we use LMSDiscreteScheduler, let's make sure latents are mulitplied by sigmas + if isinstance(self.scheduler, LMSDiscreteScheduler): + noise = noise * self.scheduler.sigmas[0].numpy() + return noise, {} + input_image, meta = preprocess(image) + moments = self.vae_encoder(input_image)[self._vae_e_output] + mean, logvar = np.split(moments, 2, axis=1) + std = np.exp(logvar * 0.5) + latents = (mean + std * np.random.randn(*mean.shape)) * 0.18215 + latents = self.scheduler.add_noise(torch.from_numpy(latents), torch.from_numpy(noise), latent_timestep).numpy() + return latents, meta + + def postprocess_image(self, image:np.ndarray, meta:Dict, output_type:str = "pil"): + """ + Postprocessing for decoded image. Takes generated image decoded by VAE decoder, unpad it to initila image size (if required), + normalize and convert to [0, 255] pixels range. Optionally, convertes it from np.ndarray to PIL.Image format + + Parameters: + image (np.ndarray): + Generated image + meta (Dict): + Metadata obtained on latents preparing step, can be empty + output_type (str, *optional*, pil): + Output format for result, can be pil or numpy + Returns: + image (List of np.ndarray or PIL.Image.Image): + Postprocessed images + """ + if "padding" in meta: + pad = meta["padding"] + (_, end_h), (_, end_w) = pad[1:3] + h, w = image.shape[2:] + unpad_h = h - end_h + unpad_w = w - end_w + image = image[:, :, :unpad_h, :unpad_w] + image = np.clip(image / 2 + 0.5, 0, 1) + image = np.transpose(image, (0, 2, 3, 1)) + # 9. Convert to PIL + if output_type == "pil": + image = self.numpy_to_pil(image) + if "src_height" in meta: + orig_height, orig_width = meta["src_height"], meta["src_width"] + image = [img.resize((orig_width, orig_height), + PIL.Image.Resampling.LANCZOS) for img in image] + else: + if "src_height" in meta: + orig_height, orig_width = meta["src_height"], meta["src_width"] + image = [cv2.resize(img, (orig_width, orig_width)) + for img in image] + return image + + def get_timesteps(self, num_inference_steps:int, strength:float): + """ + Helper function for getting scheduler timesteps for generation + In case of image-to-image generation, it updates number of steps according to strength + + Parameters: + num_inference_steps (int): + number of inference steps for generation + strength (float): + value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. + Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. + """ + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + + +select device from dropdown list for running inference using OpenVINO + +.. code:: ipython3 + + from openvino.runtime import Core + import ipywidgets as widgets + + core = Core() + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=2, options=('CPU', 'GPU', 'AUTO'), value='AUTO') + + + +Configure Inference Pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Configuration steps: 1. Load models on device 2. Configure tokenizer and +scheduler 3. Create instance of OvStableDiffusionInpaintingPipeline +class + +This can take a while to run. + +.. code:: ipython3 + + ov_config = {"INFERENCE_PRECISION_HINT": "f32"} if device.value != "CPU" else {} + + image_encoder_inpaint = core.compile_model(IMAGE_ENCODER_OV_PATH_INPAINT, device.value) + unet_model_inpaint = core.compile_model(UNET_OV_PATH_INPAINT, device.value) + vae_decoder_inpaint = core.compile_model(VAE_DECODER_OV_PATH_INPAINT, device.value, ov_config) + vae_encoder_inpaint = core.compile_model(VAE_ENCODER_OV_PATH_INPAINT, device.value, ov_config) + + ov_pipe_inpaint = OVStableDiffusionInpaintingPipeline( + image_processor=extractor, + image_encoder=image_encoder_inpaint, + unet=unet_model_inpaint, + vae_encoder=vae_encoder_inpaint, + vae_decoder=vae_decoder_inpaint, + scheduler=scheduler_inpaint, + ) + +.. code:: ipython3 + + # Code adapated from https://huggingface.co/spaces/Fantasy-Studio/Paint-by-Example/blob/main/app.py + + import os + import gradio as gr + + def predict(dict:gr.components.Image, reference:PIL.Image.Image, seed:int, step:int): + """ + This function runs when the 'paint' button is pressed. It takes 3 input images. Takes generated image decoded by VAE decoder, unpad it to initila image size (if required), + normalize and convert to [0, 255] pixels range. Optionally, convertes it from np.ndarray to PIL.Image format + + Parameters: + dict (Dict): + Contains two images in a dictionary + 'image' is the image that will be painted on + 'mask' is the black/white image specifying where to paint (white) and not to paint (black) + image (PIL.Image.Image): + Reference image that will be used by the model to know what to paint in the specified area + seed (int): + Used to initialize the random number generator state + step (int): + The number of denoising steps to run during inference. Low = fast/low quality, High = slow/higher quality + Returns: + image (PIL.Image.Image): + Postprocessed images + """ + width,height = dict["image"].size + + # If the image is not 512x512 then resize + if width < height: + factor = width / 512.0 + width = 512 + height = int((height / factor) / 8.0) * 8 + else: + factor = height / 512.0 + height = 512 + width = int((width / factor) / 8.0) * 8 + + init_image = dict["image"].convert("RGB").resize((width,height)) + mask = dict["mask"].convert("RGB").resize((width,height)) + + # If the image is not a 512x512 square then crop + if width > height: + buffer = (width - height) / 2 + input_image = init_image.crop((buffer, 0, width - buffer, 512)) + mask = mask.crop((buffer, 0, width - buffer, 512)) + elif width < height: + buffer = (height - width) / 2 + input_image = init_image.crop((0, buffer, 512, height - buffer)) + mask = mask.crop((0, buffer, 512, height - buffer)) + else: + input_image = init_image + + if not os.path.exists('output'): + os.mkdir('output') + input_image.save('output/init.png') + mask.save('output/mask.png') + reference.save('output/ref.png') + + mask = [mask] + + result = ov_pipe_inpaint( + image=input_image, + mask_image=mask, + reference_image=reference, + seed=seed, + num_inference_steps=step, + )["sample"][0] + + out_dir = Path("output") + out_dir.mkdir(exist_ok=True) + result.save('output/result.png') + + return result + + + example = {} + ref_dir = 'data/reference' + image_dir = 'data/image' + ref_list = [os.path.join(ref_dir,file) for file in os.listdir(ref_dir)] + ref_list.sort() + image_list = [os.path.join(image_dir,file) for file in os.listdir(image_dir)] + image_list.sort() + + + image_blocks = gr.Blocks() + with image_blocks as demo: + with gr.Group(): + with gr.Box(): + with gr.Row(): + with gr.Column(): + image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="pil", label="Source Image") + reference = gr.Image(source='upload', elem_id="image_upload", type="pil", label="Reference Image") + + with gr.Column(): + image_out = gr.Image(label="Output", elem_id="output-img") + steps = gr.Slider(label="Steps", value=15, minimum=2, maximum=75, step=1,interactive=True) + + seed = gr.Slider(0, 10000, label='Seed (0 = random)', value=0, step=1) + + with gr.Row(elem_id="prompt-container"): + btn = gr.Button("Paint!") + + with gr.Row(): + with gr.Column(): + gr.Examples(image_list, inputs=[image],label="Examples - Source Image",examples_per_page=12) + with gr.Column(): + gr.Examples(ref_list, inputs=[reference],label="Examples - Reference Image",examples_per_page=12) + + btn.click(fn=predict, inputs=[image, reference, seed, steps], outputs=[image_out]) + + # Launching the Gradio app + try: + image_blocks.launch(debug=False, height=680) + except Exception: + image_blocks.queue().launch(share=True, debug=False, height=680) + # if you are launching remotely, specify server_name and server_port + # image_blocks.launch(server_name='your server name', server_port='server port in int') + # Read more in the docs: https://gradio.app/docs/ + + +.. parsed-literal:: + + Running on local URL: http://127.0.0.1:7860 + + To create a public link, set `share=True` in `launch()`. + + + +.. .. raw:: html + +..
+ diff --git a/docs/notebooks/272-paint-by-example-with-output_files/272-paint-by-example.png b/docs/notebooks/272-paint-by-example-with-output_files/272-paint-by-example.png new file mode 100644 index 00000000000..6e5d7b57c37 --- /dev/null +++ b/docs/notebooks/272-paint-by-example-with-output_files/272-paint-by-example.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c19760d8edbb1566b823983afbb85c07337cb5d50476e7dd6fd602114de649 +size 3943872 diff --git a/docs/notebooks/302-pytorch-quantization-aware-training-with-output.rst b/docs/notebooks/302-pytorch-quantization-aware-training-with-output.rst index 1d30ba14ae8..4b79ee71407 100644 --- a/docs/notebooks/302-pytorch-quantization-aware-training-with-output.rst +++ b/docs/notebooks/302-pytorch-quantization-aware-training-with-output.rst @@ -26,25 +26,28 @@ notebook. Using the smaller model and dataset will speed up training and download time. To see other ResNet models, visit `PyTorch hub `__. - **NOTE**: This notebook requires a C++ compiler. + **NOTE**: This notebook requires a C++ compiler for compiling PyTorch + custom operations for quantization. For Windows we recommend to + install Visual Studio with C++ support, you can find instruction + `here `__. + For MacOS ``xcode-select --install`` command installs many developer + tools, including C++. For Linux you can install gcc with your + distribution’s package manager. **Table of contents:** - `Imports and Settings <#imports-and-settings>`__ -- `Pre-train Floating-Point - Model <#pre-train-floating-point-model>`__ +- `Pre-train Floating-Point Model <#pre-train-floating-point-model>`__ - `Train Function <#train-function>`__ - `Validate Function <#validate-function>`__ - `Helpers <#helpers>`__ - - `Get a Pre-trained FP32 - Model <#get-a-pre-trained-fp-model>`__ + - `Get a Pre-trained FP32 Model <#get-a-pre-trained-fp-model>`__ - `Create and Initialize Quantization <#create-and-initialize-quantization>`__ -- `Fine-tune the Compressed - Model <#fine-tune-the-compressed-model>`__ +- `Fine-tune the Compressed Model <#fine-tune-the-compressed-model>`__ - `Export INT8 Model to OpenVINO IR <#export-int-model-to-openvino-ir>`__ - `Benchmark Model Performance by Computing Inference @@ -64,8 +67,10 @@ hub `__. Note: you may need to restart the kernel to use updated packages. -Imports and Settings --------------------------------------------------------------- +Imports and Settings +-------------------- + + On Windows, add the required C++ directories to the system PATH. @@ -196,7 +201,7 @@ models will be stored. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/302-pytorch-quantization-aware-training/model/resnet18_fp32.pth') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/302-pytorch-quantization-aware-training/model/resnet18_fp32.pth') @@ -255,8 +260,10 @@ Download Tiny ImageNet dataset Successfully downloaded and prepared dataset at: data/tiny-imagenet-200 -Pre-train Floating-Point Model ------------------------------------------------------------------------- +Pre-train Floating-Point Model +------------------------------ + + Using NNCF for model compression assumes that a pre-trained model and a training pipeline are already in use. @@ -268,8 +275,10 @@ classes from Tiny-ImageNet. Subsequently, the training and validation functions will be reused as is for quantization-aware training. -Train Function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Train Function +~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -313,8 +322,10 @@ Train Function if i % print_frequency == 0: progress.display(i) -Validate Function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Validate Function +~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 @@ -355,8 +366,10 @@ Validate Function print(" * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}".format(top1=top1, top5=top5)) return top1.avg -Helpers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Helpers +~~~~~~~ + + .. code:: ipython3 @@ -418,8 +431,10 @@ Helpers res.append(correct_k.mul_(100.0 / batch_size)) return res -Get a Pre-trained FP32 Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Get a Pre-trained FP32 Model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + А pre-trained floating-point model is a prerequisite for quantization. It can be obtained by tuning from scratch with the code below. However, @@ -485,9 +500,9 @@ section at the top of this notebook. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead. warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`. warnings.warn(msg) @@ -545,8 +560,10 @@ benchmark it in comparison with the ``INT8`` model. FP32 model was exported to model/resnet18_fp32.xml. -Create and Initialize Quantization ----------------------------------------------------------------------------- +Create and Initialize Quantization +---------------------------------- + + NNCF enables compression-aware training by integrating into regular training pipelines. The framework is designed so that modifications to @@ -584,10 +601,10 @@ scenario and requires only 3 modifications. .. parsed-literal:: - 2023-10-31 00:15:43.733728: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2023-10-31 00:15:43.767038: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2023-12-07 00:28:30.511089: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-12-07 00:28:30.544997: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2023-10-31 00:15:44.314131: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2023-12-07 00:28:31.224999: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Evaluate the new model on the validation set after initialization of @@ -603,20 +620,22 @@ demonstrated here. .. parsed-literal:: - Test: [ 0/79] Time 0.168 (0.168) Loss 0.981 (0.981) Acc@1 78.91 (78.91) Acc@5 89.84 (89.84) - Test: [10/79] Time 0.156 (0.165) Loss 1.905 (1.623) Acc@1 46.88 (60.51) Acc@5 82.03 (84.09) - Test: [20/79] Time 0.155 (0.160) Loss 1.734 (1.692) Acc@1 63.28 (58.63) Acc@5 79.69 (83.04) - Test: [30/79] Time 0.154 (0.157) Loss 2.282 (1.781) Acc@1 50.00 (57.31) Acc@5 69.53 (81.50) - Test: [40/79] Time 0.152 (0.156) Loss 1.540 (1.825) Acc@1 62.50 (55.83) Acc@5 85.94 (80.96) - Test: [50/79] Time 0.152 (0.156) Loss 1.972 (1.820) Acc@1 57.03 (56.05) Acc@5 75.00 (80.73) - Test: [60/79] Time 0.152 (0.156) Loss 1.731 (1.846) Acc@1 57.81 (55.51) Acc@5 85.16 (80.21) - Test: [70/79] Time 0.154 (0.155) Loss 2.412 (1.872) Acc@1 47.66 (55.15) Acc@5 71.88 (79.61) + Test: [ 0/79] Time 0.167 (0.167) Loss 0.981 (0.981) Acc@1 78.91 (78.91) Acc@5 89.84 (89.84) + Test: [10/79] Time 0.149 (0.151) Loss 1.905 (1.623) Acc@1 46.88 (60.51) Acc@5 82.03 (84.09) + Test: [20/79] Time 0.150 (0.151) Loss 1.734 (1.692) Acc@1 63.28 (58.63) Acc@5 79.69 (83.04) + Test: [30/79] Time 0.149 (0.150) Loss 2.282 (1.781) Acc@1 50.00 (57.31) Acc@5 69.53 (81.50) + Test: [40/79] Time 0.148 (0.150) Loss 1.540 (1.825) Acc@1 62.50 (55.83) Acc@5 85.94 (80.96) + Test: [50/79] Time 0.147 (0.150) Loss 1.972 (1.820) Acc@1 57.03 (56.05) Acc@5 75.00 (80.73) + Test: [60/79] Time 0.148 (0.150) Loss 1.731 (1.846) Acc@1 57.81 (55.51) Acc@5 85.16 (80.21) + Test: [70/79] Time 0.164 (0.151) Loss 2.412 (1.872) Acc@1 47.66 (55.15) Acc@5 71.88 (79.61) * Acc@1 55.540 Acc@5 80.200 Accuracy of initialized INT8 model: 55.540 -Fine-tune the Compressed Model ------------------------------------------------------------------------- +Fine-tune the Compressed Model +------------------------------ + + At this step, a regular fine-tuning process is applied to further improve quantized model accuracy. Normally, several epochs of tuning are @@ -641,37 +660,39 @@ training pipeline are required. Here is a simple example. .. parsed-literal:: - Epoch:[0][ 0/782] Time 0.412 (0.412) Loss 0.740 (0.740) Acc@1 84.38 (84.38) Acc@5 96.88 (96.88) - Epoch:[0][ 50/782] Time 0.383 (0.387) Loss 0.911 (0.802) Acc@1 78.91 (80.15) Acc@5 92.97 (94.42) - Epoch:[0][100/782] Time 0.387 (0.388) Loss 0.631 (0.798) Acc@1 84.38 (80.24) Acc@5 95.31 (94.38) - Epoch:[0][150/782] Time 0.381 (0.388) Loss 0.836 (0.792) Acc@1 80.47 (80.48) Acc@5 94.53 (94.43) - Epoch:[0][200/782] Time 0.369 (0.386) Loss 0.873 (0.780) Acc@1 75.00 (80.65) Acc@5 94.53 (94.59) - Epoch:[0][250/782] Time 0.385 (0.386) Loss 0.735 (0.778) Acc@1 84.38 (80.77) Acc@5 95.31 (94.53) - Epoch:[0][300/782] Time 0.368 (0.386) Loss 0.615 (0.771) Acc@1 85.16 (80.99) Acc@5 97.66 (94.58) - Epoch:[0][350/782] Time 0.392 (0.386) Loss 0.599 (0.767) Acc@1 85.16 (81.14) Acc@5 95.31 (94.58) - Epoch:[0][400/782] Time 0.382 (0.386) Loss 0.798 (0.765) Acc@1 82.03 (81.21) Acc@5 92.97 (94.56) - Epoch:[0][450/782] Time 0.377 (0.386) Loss 0.630 (0.762) Acc@1 85.16 (81.26) Acc@5 96.88 (94.58) - Epoch:[0][500/782] Time 0.367 (0.386) Loss 0.633 (0.757) Acc@1 85.94 (81.45) Acc@5 96.88 (94.63) - Epoch:[0][550/782] Time 0.406 (0.386) Loss 0.749 (0.755) Acc@1 82.03 (81.49) Acc@5 92.97 (94.65) - Epoch:[0][600/782] Time 0.397 (0.385) Loss 0.927 (0.753) Acc@1 78.12 (81.53) Acc@5 88.28 (94.67) - Epoch:[0][650/782] Time 0.392 (0.385) Loss 0.645 (0.749) Acc@1 84.38 (81.60) Acc@5 95.31 (94.71) - Epoch:[0][700/782] Time 0.399 (0.386) Loss 0.816 (0.749) Acc@1 82.03 (81.62) Acc@5 91.41 (94.69) - Epoch:[0][750/782] Time 0.404 (0.386) Loss 0.811 (0.746) Acc@1 80.47 (81.69) Acc@5 94.53 (94.72) - Test: [ 0/79] Time 0.166 (0.166) Loss 1.092 (1.092) Acc@1 75.00 (75.00) Acc@5 86.72 (86.72) - Test: [10/79] Time 0.150 (0.140) Loss 1.917 (1.526) Acc@1 48.44 (62.64) Acc@5 78.12 (83.88) - Test: [20/79] Time 0.137 (0.138) Loss 1.631 (1.602) Acc@1 64.06 (60.68) Acc@5 81.25 (83.71) - Test: [30/79] Time 0.136 (0.138) Loss 2.037 (1.691) Acc@1 57.81 (59.25) Acc@5 71.09 (82.23) - Test: [40/79] Time 0.134 (0.137) Loss 1.563 (1.743) Acc@1 64.84 (58.02) Acc@5 82.81 (81.33) - Test: [50/79] Time 0.137 (0.137) Loss 1.926 (1.750) Acc@1 52.34 (57.77) Acc@5 76.56 (81.04) - Test: [60/79] Time 0.137 (0.137) Loss 1.559 (1.781) Acc@1 67.19 (57.24) Acc@5 84.38 (80.58) - Test: [70/79] Time 0.137 (0.137) Loss 2.353 (1.806) Acc@1 46.88 (56.81) Acc@5 72.66 (80.08) + Epoch:[0][ 0/782] Time 0.407 (0.407) Loss 0.740 (0.740) Acc@1 84.38 (84.38) Acc@5 96.88 (96.88) + Epoch:[0][ 50/782] Time 0.383 (0.393) Loss 0.911 (0.802) Acc@1 78.91 (80.15) Acc@5 92.97 (94.42) + Epoch:[0][100/782] Time 0.406 (0.393) Loss 0.631 (0.798) Acc@1 84.38 (80.24) Acc@5 95.31 (94.38) + Epoch:[0][150/782] Time 0.408 (0.392) Loss 0.836 (0.792) Acc@1 80.47 (80.48) Acc@5 94.53 (94.43) + Epoch:[0][200/782] Time 0.413 (0.395) Loss 0.873 (0.780) Acc@1 75.00 (80.65) Acc@5 94.53 (94.59) + Epoch:[0][250/782] Time 0.384 (0.395) Loss 0.735 (0.778) Acc@1 84.38 (80.77) Acc@5 95.31 (94.53) + Epoch:[0][300/782] Time 0.372 (0.395) Loss 0.615 (0.771) Acc@1 85.16 (80.99) Acc@5 97.66 (94.58) + Epoch:[0][350/782] Time 0.398 (0.394) Loss 0.599 (0.767) Acc@1 85.16 (81.14) Acc@5 95.31 (94.58) + Epoch:[0][400/782] Time 0.386 (0.394) Loss 0.798 (0.765) Acc@1 82.03 (81.21) Acc@5 92.97 (94.56) + Epoch:[0][450/782] Time 0.397 (0.394) Loss 0.630 (0.762) Acc@1 85.16 (81.26) Acc@5 96.88 (94.58) + Epoch:[0][500/782] Time 0.368 (0.393) Loss 0.633 (0.757) Acc@1 85.94 (81.45) Acc@5 96.88 (94.63) + Epoch:[0][550/782] Time 0.416 (0.393) Loss 0.749 (0.755) Acc@1 82.03 (81.49) Acc@5 92.97 (94.65) + Epoch:[0][600/782] Time 0.388 (0.393) Loss 0.927 (0.753) Acc@1 78.12 (81.53) Acc@5 88.28 (94.67) + Epoch:[0][650/782] Time 0.424 (0.392) Loss 0.645 (0.749) Acc@1 84.38 (81.60) Acc@5 95.31 (94.71) + Epoch:[0][700/782] Time 0.376 (0.392) Loss 0.816 (0.749) Acc@1 82.03 (81.62) Acc@5 91.41 (94.69) + Epoch:[0][750/782] Time 0.362 (0.392) Loss 0.811 (0.746) Acc@1 80.47 (81.69) Acc@5 94.53 (94.72) + Test: [ 0/79] Time 0.157 (0.157) Loss 1.092 (1.092) Acc@1 75.00 (75.00) Acc@5 86.72 (86.72) + Test: [10/79] Time 0.139 (0.141) Loss 1.917 (1.526) Acc@1 48.44 (62.64) Acc@5 78.12 (83.88) + Test: [20/79] Time 0.135 (0.140) Loss 1.631 (1.602) Acc@1 64.06 (60.68) Acc@5 81.25 (83.71) + Test: [30/79] Time 0.133 (0.138) Loss 2.037 (1.691) Acc@1 57.81 (59.25) Acc@5 71.09 (82.23) + Test: [40/79] Time 0.138 (0.138) Loss 1.563 (1.743) Acc@1 64.84 (58.02) Acc@5 82.81 (81.33) + Test: [50/79] Time 0.136 (0.138) Loss 1.926 (1.750) Acc@1 52.34 (57.77) Acc@5 76.56 (81.04) + Test: [60/79] Time 0.139 (0.137) Loss 1.559 (1.781) Acc@1 67.19 (57.24) Acc@5 84.38 (80.58) + Test: [70/79] Time 0.140 (0.138) Loss 2.353 (1.806) Acc@1 46.88 (56.81) Acc@5 72.66 (80.08) * Acc@1 57.320 Acc@5 80.730 Accuracy of tuned INT8 model: 57.320 Accuracy drop of tuned INT8 model over pre-trained FP32 model: -1.800 -Export INT8 Model to OpenVINO IR --------------------------------------------------------------------------- +Export INT8 Model to OpenVINO IR +-------------------------------- + + .. code:: ipython3 @@ -690,8 +711,10 @@ Export INT8 Model to OpenVINO IR INT8 Omodel exported to model/resnet18_int8.xml. -Benchmark Model Performance by Computing Inference Time -------------------------------------------------------------------------------------------------- +Benchmark Model Performance by Computing Inference Time +------------------------------------------------------- + + Finally, measure the inference performance of the ``FP32`` and ``INT8`` models, using `Benchmark @@ -729,9 +752,9 @@ throughput (frames per second) values. .. parsed-literal:: Benchmark FP32 model (IR) - [ INFO ] Throughput: 2952.65 FPS + [ INFO ] Throughput: 2930.00 FPS Benchmark INT8 model (IR) - [ INFO ] Throughput: 11986.34 FPS + [ INFO ] Throughput: 11839.41 FPS Show CPU Information for reference. diff --git a/docs/notebooks/406-3D-pose-estimation-with-output.rst b/docs/notebooks/406-3D-pose-estimation-with-output.rst index 851fbbbc926..c55447a9e76 100644 --- a/docs/notebooks/406-3D-pose-estimation-with-output.rst +++ b/docs/notebooks/406-3D-pose-estimation-with-output.rst @@ -8,9 +8,11 @@ from `Open Model Zoo `__. At the end of this notebook, you will see live inference results from your webcam (if available). Alternatively, you can also upload a video file to test -out the algorithms. **Make sure you have properly installed the** -`Jupyter extension `__ -**and been using JupyterLab to run the demo as suggested in the README.md** +out the algorithms. **Make sure you have properly installed +the**\ `Jupyter +extension `__\ **and +been using JupyterLab to run the demo as suggested in the +``README.md``** **NOTE**: *To use a webcam, you must run this Jupyter notebook on a computer with a webcam. If you run on a remote server, the webcam @@ -49,10 +51,12 @@ Windows: Chrome* *macOS: Safari* - `Run Pose Estimation on a Video File <#run-pose-estimation-on-a-video-file>`__ -Prerequisites -------------------------------------------------------- +Prerequisites +------------- -**The "pythreejs" extension may not display properly when using the + + +**The ``pythreejs`` extension may not display properly when using the latest Jupyter Notebook release (2.4.1). Therefore, it is recommended to use Jupyter Lab instead.** @@ -65,61 +69,75 @@ use Jupyter Lab instead.** Collecting pythreejs Using cached pythreejs-2.4.2-py3-none-any.whl (3.4 MB) - Requirement already satisfied: openvino-dev==2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2023.1.0) - Requirement already satisfied: addict>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2.4.0) - Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (0.7.1) - Requirement already satisfied: jstyleson>=0.0.2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (0.0.2) - Requirement already satisfied: networkx<=3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2.8.2) - Requirement already satisfied: numpy>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (1.24.3) - Requirement already satisfied: opencv-python in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (4.8.1.78) - Requirement already satisfied: openvino-telemetry>=2022.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2023.2.1) - Requirement already satisfied: pillow>=8.1.2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (10.0.1) - Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (6.0.1) - Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2.31.0) - Requirement already satisfied: texttable>=1.6.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (1.7.0) - Requirement already satisfied: tqdm>=4.54.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (4.66.1) - Requirement already satisfied: openvino==2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2023.1.0) - Requirement already satisfied: scipy<1.11,>=1.8 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (1.10.1) - Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.1) + Collecting openvino-dev==2023.1.0 + Using cached openvino_dev-2023.1.0-12185-py3-none-any.whl.metadata (17 kB) + Requirement already satisfied: addict>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2.4.0) + Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (0.7.1) + Requirement already satisfied: jstyleson>=0.0.2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (0.0.2) + Requirement already satisfied: networkx<=3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2.8.2) + Requirement already satisfied: numpy>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (1.23.5) + Requirement already satisfied: opencv-python in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (4.8.1.78) + Requirement already satisfied: openvino-telemetry>=2022.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2023.2.1) + Requirement already satisfied: pillow>=8.1.2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (10.1.0) + Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (6.0.1) + Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (2.31.0) + Requirement already satisfied: texttable>=1.6.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (1.7.0) + Requirement already satisfied: tqdm>=4.54.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (4.66.1) + Collecting openvino==2023.1.0 (from openvino-dev==2023.1.0) + Using cached openvino-2023.1.0-12185-cp38-cp38-manylinux2014_x86_64.whl.metadata (6.3 kB) + Requirement already satisfied: scipy<1.11,>=1.8 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev==2023.1.0) (1.10.1) + Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.1) Collecting ipydatawidgets>=1.1.1 (from pythreejs) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl.metadata (1.4 kB) - Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.13.0) + Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.0) Collecting traittypes>=0.2.0 (from ipydatawidgets>=1.1.1->pythreejs) Using cached traittypes-0.2.1-py2.py3-none-any.whl (8.6 kB) - Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.1.4) - Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) - Requirement already satisfied: widgetsnbextension~=4.0.9 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.9) - Requirement already satisfied: jupyterlab-widgets~=3.0.9 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.9) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (3.3.1) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (3.4) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (2.0.7) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (2023.7.22) - Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) - Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) - Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) - Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.6) - Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) - Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.39) - Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.16.1) - Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) - Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.5.0) - Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.8.0) - Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.3) - Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) - Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.9) - Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.0.1) - Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) - Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) + Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.0) + Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) + Requirement already satisfied: widgetsnbextension~=4.0.9 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.9) + Requirement already satisfied: jupyterlab-widgets~=3.0.9 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.9) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (3.3.2) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (3.6) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (2.1.0) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev==2023.1.0) (2023.11.17) + Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) + Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) + Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) + Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.6) + Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) + Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.41) + Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.17.2) + Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) + Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.8.0) + Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) + Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.3) + Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) + Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.12) + Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.0.1) + Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) + Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) + Using cached openvino_dev-2023.1.0-12185-py3-none-any.whl (5.8 MB) + Using cached openvino-2023.1.0-12185-cp38-cp38-manylinux2014_x86_64.whl (35.2 MB) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl (271 kB) DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - Installing collected packages: traittypes, ipydatawidgets, pythreejs - Successfully installed ipydatawidgets-4.3.5 pythreejs-2.4.2 traittypes-0.2.1 + Installing collected packages: traittypes, openvino, openvino-dev, ipydatawidgets, pythreejs + Attempting uninstall: openvino + Found existing installation: openvino 2023.2.0 + Uninstalling openvino-2023.2.0: + Successfully uninstalled openvino-2023.2.0 + Attempting uninstall: openvino-dev + Found existing installation: openvino-dev 2023.2.0 + Uninstalling openvino-dev-2023.2.0: + Successfully uninstalled openvino-dev-2023.2.0 + Successfully installed ipydatawidgets-4.3.5 openvino-2023.1.0 openvino-dev-2023.1.0 pythreejs-2.4.2 traittypes-0.2.1 Note: you may need to restart the kernel to use updated packages. -Imports -------------------------------------------------- +Imports +------- + + .. code:: ipython3 @@ -141,11 +159,15 @@ Imports import engine.engine3js as engine from engine.parse_poses import parse_poses -The model ---------------------------------------------------- +The model +--------- + + + +Download the model +~~~~~~~~~~~~~~~~~~ + -Download the model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We use ``omz_downloader``, which is a command line tool from the ``openvino-dev`` package. ``omz_downloader`` automatically creates a @@ -186,8 +208,10 @@ directory structure and downloads the selected model. -Convert Model to OpenVINO IR format -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Convert Model to OpenVINO IR format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The selected model comes from the public directory, which means it must be converted into OpenVINO Intermediate Representation (OpenVINO IR). We @@ -210,23 +234,25 @@ IR format. .. parsed-literal:: ========== Converting human-pose-estimation-3d-0001 to ONNX - Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-path=model/public/human-pose-estimation-3d-0001 --model-name=PoseEstimationWithMobileNet --model-param=is_convertible_by_mo=True --import-module=model --weights=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.pth --input-shape=1,3,256,448 --input-names=data --output-names=features,heatmaps,pafs --output-file=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx + Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-path=model/public/human-pose-estimation-3d-0001 --model-name=PoseEstimationWithMobileNet --model-param=is_convertible_by_mo=True --import-module=model --weights=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.pth --input-shape=1,3,256,448 --input-names=data --output-names=features,heatmaps,pafs --output-file=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx ONNX check passed successfully. ========== Converting human-pose-estimation-3d-0001 to IR (FP32) - Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/human-pose-estimation-3d-0001/FP32 --model_name=human-pose-estimation-3d-0001 --input=data '--mean_values=data[128.0,128.0,128.0]' '--scale_values=data[255.0,255.0,255.0]' --output=features,heatmaps,pafs --input_model=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 256, 448]' --compress_to_fp16=False + Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/human-pose-estimation-3d-0001/FP32 --model_name=human-pose-estimation-3d-0001 --input=data '--mean_values=data[128.0,128.0,128.0]' '--scale_values=data[255.0,255.0,255.0]' --output=features,heatmaps,pafs --input_model=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 256, 448]' --compress_to_fp16=False [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html [ SUCCESS ] Generated IR version 11 model. - [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/406-3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.xml - [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/406-3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.bin + [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/406-3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.xml + [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/406-3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.bin -Select inference device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + select device from dropdown list for running inference using OpenVINO @@ -252,8 +278,10 @@ select device from dropdown list for running inference using OpenVINO -Load the model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load the model +~~~~~~~~~~~~~~ + + Converted models are located in a fixed structure, which indicates vendor, model name and precision. @@ -294,11 +322,15 @@ heat maps, PAF (part affinity fields) and features. -Processing ----------------------------------------------------- +Processing +---------- + + + +Model Inference +~~~~~~~~~~~~~~~ + -Model Inference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Frames captured from video files or the live webcam are used as the input for the 3D model. This is how you obtain the output heat maps, PAF @@ -335,8 +367,10 @@ input for the 3D model. This is how you obtain the output heat maps, PAF return results -Draw 2D Pose Overlays -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Draw 2D Pose Overlays +~~~~~~~~~~~~~~~~~~~~~ + + We need to define some connections between the joints in advance, so that we can draw the structure of the human body in the resulting image @@ -417,8 +451,10 @@ from Open Model Zoo. return frame -Main Processing Function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Main Processing Function +~~~~~~~~~~~~~~~~~~~~~~~~ + + Run 3D pose estimation on the specified source. It could be either a webcam feed or a video file. @@ -581,11 +617,15 @@ webcam feed or a video file. if skeleton_set: engine3D.scene_remove(skeleton_set) -Run ---------------------------------------------- +Run +--- + + + +Run Live Pose Estimation +~~~~~~~~~~~~~~~~~~~~~~~~ + -Run Live Pose Estimation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Run, using a webcam as the video input. By default, the primary webcam is set with ``source=0``. If you have multiple webcams, each one will be @@ -610,8 +650,10 @@ picture on the left to interact. run_pose_estimation(source=0, flip=True, use_popup=False) -Run Pose Estimation on a Video File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Run Pose Estimation on a Video File +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + If you do not have a webcam, you can still run this demo with a video file. Any `format supported by diff --git a/docs/notebooks/all_notebooks_paths.txt b/docs/notebooks/all_notebooks_paths.txt index f5932dc140a..030465414f9 100644 --- a/docs/notebooks/all_notebooks_paths.txt +++ b/docs/notebooks/all_notebooks_paths.txt @@ -95,6 +95,7 @@ notebooks/244-named-entity-recognition/244-named-entity-recognition.ipynb notebooks/245-typo-detector/245-typo-detector.ipynb notebooks/246-depth-estimation-videpth/246-depth-estimation-videpth.ipynb notebooks/247-code-language-id/247-code-language-id.ipynb +notebooks/248-stable-diffusion-xl/248-ssd-b1.ipynb notebooks/248-stable-diffusion-xl/248-stable-diffusion-xl.ipynb notebooks/249-oneformer-segmentation/249-oneformer-segmentation.ipynb notebooks/250-music-generation/250-music-generation.ipynb @@ -111,12 +112,16 @@ notebooks/260-pix2struct-docvqa/260-pix2struct-docvqa.ipynb notebooks/261-fast-segment-anything/261-fast-segment-anything.ipynb notebooks/262-softvc-voice-conversion/262-softvc-voice-conversion.ipynb notebooks/263-latent-consistency-models-image-generation/263-latent-consistency-models-image-generation.ipynb +notebooks/263-latent-consistency-models-image-generation/263-lcm-lora-controlnet.ipynb notebooks/264-qrcode-monster/264-qrcode-monster.ipynb notebooks/265-wuerstchen-image-generation/265-wuerstchen-image-generation.ipynb notebooks/266-speculative-sampling/266-speculative-sampling.ipynb notebooks/267-distil-whisper-asr/267-distil-whisper-asr.ipynb notebooks/268-table-question-answering/268-table-question-answering.ipynb notebooks/269-film-slowmo/269-film-slowmo.ipynb +notebooks/270-sound-generation-audioldm2/270-sound-generation-audioldm2.ipynb +notebooks/271-sdxl-turbo/271-sdxl-turbo.ipynb +notebooks/272-paint-by-example/272-paint-by-example.ipynb notebooks/301-tensorflow-training-openvino/301-tensorflow-training-openvino.ipynb notebooks/301-tensorflow-training-openvino/301-tensorflow-training-openvino-nncf.ipynb notebooks/302-pytorch-quantization-aware-training/302-pytorch-quantization-aware-training.ipynb diff --git a/docs/notebooks/index.html b/docs/notebooks/index.html index 9ee8ff81125..7e753ff14f6 100644 --- a/docs/notebooks/index.html +++ b/docs/notebooks/index.html @@ -1,207 +1,229 @@ -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/


../
-001-hello-world-with-output_files/                 31-Oct-2023 00:35                   -
-003-hello-segmentation-with-output_files/          31-Oct-2023 00:35                   -
-004-hello-detection-with-output_files/             31-Oct-2023 00:35                   -
-101-tensorflow-classification-to-openvino-with-..> 31-Oct-2023 00:35                   -
-102-pytorch-onnx-to-openvino-with-output_files/    31-Oct-2023 00:35                   -
-102-pytorch-to-openvino-with-output_files/         31-Oct-2023 00:35                   -
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35                   -
-106-auto-device-with-output_files/                 31-Oct-2023 00:35                   -
-109-latency-tricks-with-output_files/              31-Oct-2023 00:35                   -
-109-throughput-tricks-with-output_files/           31-Oct-2023 00:35                   -
-110-ct-scan-live-inference-with-output_files/      31-Oct-2023 00:35                   -
-110-ct-segmentation-quantize-nncf-with-output_f..> 31-Oct-2023 00:35                   -
-111-yolov5-quantization-migration-with-output_f..> 31-Oct-2023 00:35                   -
-113-image-classification-quantization-with-outp..> 31-Oct-2023 00:35                   -
-115-async-api-with-output_files/                   31-Oct-2023 00:35                   -
-117-model-server-with-output_files/                31-Oct-2023 00:35                   -
-118-optimize-preprocessing-with-output_files/      31-Oct-2023 00:35                   -
-119-tflite-to-openvino-with-output_files/          31-Oct-2023 00:35                   -
-120-tensorflow-instance-segmentation-to-openvin..> 31-Oct-2023 00:35                   -
-120-tensorflow-object-detection-to-openvino-wit..> 31-Oct-2023 00:35                   -
-123-detectron2-to-openvino-with-output_files/      31-Oct-2023 00:35                   -
-125-convnext-classification-with-output_files/     31-Oct-2023 00:35                   -
-126-tensorflow-hub-with-output_files/              31-Oct-2023 00:35                   -
-201-vision-monodepth-with-output_files/            31-Oct-2023 00:35                   -
-202-vision-superresolution-image-with-output_files 31-Oct-2023 00:35                   -
-203-meter-reader-with-output_files/                31-Oct-2023 00:35                   -
-204-segmenter-semantic-segmentation-with-output..> 31-Oct-2023 00:35                   -
-205-vision-background-removal-with-output_files/   31-Oct-2023 00:35                   -
-206-vision-paddlegan-anime-with-output_files/      31-Oct-2023 00:35                   -
-207-vision-paddlegan-superresolution-with-outpu..> 31-Oct-2023 00:35                   -
-208-optical-character-recognition-with-output_f..> 31-Oct-2023 00:35                   -
-209-handwritten-ocr-with-output_files/             31-Oct-2023 00:35                   -
-211-speech-to-text-with-output_files/              31-Oct-2023 00:35                   -
-212-pyannote-speaker-diarization-with-output_files 31-Oct-2023 00:35                   -
-215-image-inpainting-with-output_files/            31-Oct-2023 00:35                   -
-217-vision-deblur-with-output_files/               31-Oct-2023 00:35                   -
-218-vehicle-detection-and-recognition-with-outp..> 31-Oct-2023 00:35                   -
-220-cross-lingual-books-alignment-with-output_f..> 31-Oct-2023 00:35                   -
-222-vision-image-colorization-with-output_files/   31-Oct-2023 00:35                   -
-224-3D-segmentation-point-clouds-with-output_files 31-Oct-2023 00:35                   -
-225-stable-diffusion-text-to-image-with-output_..> 31-Oct-2023 00:35                   -
-226-yolov7-optimization-with-output_files/         31-Oct-2023 00:35                   -
-228-clip-zero-shot-convert-with-output_files/      31-Oct-2023 00:35                   -
-228-clip-zero-shot-quantize-with-output_files/     31-Oct-2023 00:35                   -
-230-yolov8-instance-segmentation-with-output_files 31-Oct-2023 00:35                   -
-230-yolov8-keypoint-detection-with-output_files/   31-Oct-2023 00:35                   -
-230-yolov8-object-detection-with-output_files/     31-Oct-2023 00:35                   -
-231-instruct-pix2pix-image-editing-with-output_..> 31-Oct-2023 00:35                   -
-232-clip-language-saliency-map-with-output_files/  31-Oct-2023 00:35                   -
-233-blip-convert-with-output_files/                31-Oct-2023 00:35                   -
-233-blip-optimize-with-output_files/               31-Oct-2023 00:35                   -
-234-encodec-audio-compression-with-output_files/   31-Oct-2023 00:35                   -
-235-controlnet-stable-diffusion-with-output_files/ 31-Oct-2023 00:35                   -
-236-stable-diffusion-v2-optimum-demo-comparison..> 31-Oct-2023 00:35                   -
-236-stable-diffusion-v2-optimum-demo-with-outpu..> 31-Oct-2023 00:35                   -
-236-stable-diffusion-v2-text-to-image-demo-with..> 31-Oct-2023 00:35                   -
-237-segment-anything-with-output_files/            31-Oct-2023 00:35                   -
-238-deep-floyd-if-optimize-with-output_files/      31-Oct-2023 00:35                   -
-239-image-bind-convert-with-output_files/          31-Oct-2023 00:35                   -
-241-riffusion-text-to-music-with-output_files/     31-Oct-2023 00:35                   -
-243-tflite-selfie-segmentation-with-output_files/  31-Oct-2023 00:35                   -
-246-depth-estimation-videpth-with-output_files/    31-Oct-2023 00:35                   -
-248-stable-diffusion-xl-with-output_files/         31-Oct-2023 00:35                   -
-249-oneformer-segmentation-with-output_files/      31-Oct-2023 00:35                   -
-251-tiny-sd-image-generation-with-output_files/    31-Oct-2023 00:35                   -
-257-llava-multimodal-chatbot-with-output_files/    31-Oct-2023 00:35                   -
-258-blip-diffusion-subject-generation-with-outp..> 31-Oct-2023 00:35                   -
-259-decidiffusion-image-generation-with-output_..> 31-Oct-2023 00:35                   -
-260-pix2struct-docvqa-with-output_files/           31-Oct-2023 00:35                   -
-261-fast-segment-anything-with-output_files/       31-Oct-2023 00:35                   -
-263-latent-consistency-models-image-generation-..> 31-Oct-2023 00:35                   -
-301-tensorflow-training-openvino-nncf-with-outp..> 31-Oct-2023 00:35                   -
-301-tensorflow-training-openvino-with-output_files 31-Oct-2023 00:35                   -
-305-tensorflow-quantization-aware-training-with..> 31-Oct-2023 00:35                   -
-401-object-detection-with-output_files/            31-Oct-2023 00:35                   -
-402-pose-estimation-with-output_files/             31-Oct-2023 00:35                   -
-403-action-recognition-webcam-with-output_files/   31-Oct-2023 00:35                   -
-404-style-transfer-with-output_files/              31-Oct-2023 00:35                   -
-405-paddle-ocr-webcam-with-output_files/           31-Oct-2023 00:35                   -
-407-person-tracking-with-output_files/             31-Oct-2023 00:35                   -
-notebook_utils-with-output_files/                  31-Oct-2023 00:35                   -
-001-hello-world-with-output.rst                    31-Oct-2023 00:35                5528
-002-openvino-api-with-output.rst                   31-Oct-2023 00:35               31911
-003-hello-segmentation-with-output.rst             31-Oct-2023 00:35                7250
-004-hello-detection-with-output.rst                31-Oct-2023 00:35                8354
-101-tensorflow-classification-to-openvino-with-..> 31-Oct-2023 00:35               10299
-102-pytorch-onnx-to-openvino-with-output.rst       31-Oct-2023 00:35               20461
-102-pytorch-to-openvino-with-output.rst            31-Oct-2023 00:35               25916
-103-paddle-to-openvino-classification-with-outp..> 31-Oct-2023 00:35               17486
-104-model-tools-with-output.rst                    31-Oct-2023 00:35               21502
-105-language-quantize-bert-with-output.rst         31-Oct-2023 00:35               20417
-106-auto-device-with-output.rst                    31-Oct-2023 00:35               24736
-107-speech-recognition-quantization-data2vec-wi..> 31-Oct-2023 00:35              970503
-108-gpu-device-with-output.rst                     31-Oct-2023 00:35               56803
-109-latency-tricks-with-output.rst                 31-Oct-2023 00:35               24713
-109-throughput-tricks-with-output.rst              31-Oct-2023 00:35               26977
-110-ct-scan-live-inference-with-output.rst         31-Oct-2023 00:35               18376
-110-ct-segmentation-quantize-nncf-with-output.rst  31-Oct-2023 00:35               38113
-111-yolov5-quantization-migration-with-output.rst  31-Oct-2023 00:35               50833
-112-pytorch-post-training-quantization-nncf-wit..> 31-Oct-2023 00:35               29881
-113-image-classification-quantization-with-outp..> 31-Oct-2023 00:35               22860
-115-async-api-with-output.rst                      31-Oct-2023 00:35               20748
-116-sparsity-optimization-with-output.rst          31-Oct-2023 00:35               18408
-117-model-server-with-output.rst                   31-Oct-2023 00:35               22391
-118-optimize-preprocessing-with-output.rst         31-Oct-2023 00:35               21598
-119-tflite-to-openvino-with-output.rst             31-Oct-2023 00:35               11796
-120-tensorflow-instance-segmentation-to-openvin..> 31-Oct-2023 00:35               25913
-120-tensorflow-object-detection-to-openvino-wit..> 31-Oct-2023 00:35               28132
-121-convert-to-openvino-with-output.rst            31-Oct-2023 00:35               90175
-122-speech-recognition-quantization-wav2vec2-wi..> 31-Oct-2023 00:35              482510
-122-yolov8-quantization-with-accuracy-control-w..> 31-Oct-2023 00:35               21676
-123-detectron2-to-openvino-with-output.rst         31-Oct-2023 00:35               16703
-124-hugging-face-hub-with-output.rst               31-Oct-2023 00:35               16482
-125-convnext-classification-with-output.rst        31-Oct-2023 00:35               10593
-126-tensorflow-hub-with-output.rst                 31-Oct-2023 00:35               15512
-201-vision-monodepth-with-output.rst               31-Oct-2023 00:35              969651
-202-vision-superresolution-image-with-output.rst   31-Oct-2023 00:35               28083
-202-vision-superresolution-video-with-output.rst   31-Oct-2023 00:35             9642402
-203-meter-reader-with-output.rst                   31-Oct-2023 00:35               27185
-204-segmenter-semantic-segmentation-with-output..> 31-Oct-2023 00:35               28012
-205-vision-background-removal-with-output.rst      31-Oct-2023 00:35               14676
-206-vision-paddlegan-anime-with-output.rst         31-Oct-2023 00:35               22317
-207-vision-paddlegan-superresolution-with-outpu..> 31-Oct-2023 00:35               18189
-208-optical-character-recognition-with-output.rst  31-Oct-2023 00:35               28750
-209-handwritten-ocr-with-output.rst                31-Oct-2023 00:35               13321
-210-slowfast-video-recognition-with-output.rst     31-Oct-2023 00:35              769241
-211-speech-to-text-with-output.rst                 31-Oct-2023 00:35               89249
-212-pyannote-speaker-diarization-with-output.rst   31-Oct-2023 00:35             1294876
-213-question-answering-with-output.rst             31-Oct-2023 00:35               23057
-214-grammar-correction-with-output.rst             31-Oct-2023 00:35               32428
-215-image-inpainting-with-output.rst               31-Oct-2023 00:35               10044
-217-vision-deblur-with-output.rst                  31-Oct-2023 00:35               12552
-218-vehicle-detection-and-recognition-with-outp..> 31-Oct-2023 00:35               18290
-219-knowledge-graphs-conve-with-output.rst         31-Oct-2023 00:35               26242
-220-cross-lingual-books-alignment-with-output.rst  31-Oct-2023 00:35               53274
-221-machine-translation-with-output.rst            31-Oct-2023 00:35               10182
-222-vision-image-colorization-with-output.rst      31-Oct-2023 00:35               19493
-224-3D-segmentation-point-clouds-with-output.rst   31-Oct-2023 00:35                9960
-225-stable-diffusion-text-to-image-with-output.rst 31-Oct-2023 00:35               56071
-226-yolov7-optimization-with-output.rst            31-Oct-2023 00:35               45974
-227-whisper-convert-with-output.rst                31-Oct-2023 00:35               18077
-227-whisper-nncf-quantize-with-output.rst          31-Oct-2023 00:35               21850
-228-clip-zero-shot-convert-with-output.rst         31-Oct-2023 00:35               15239
-228-clip-zero-shot-quantize-with-output.rst        31-Oct-2023 00:35               15739
-229-distilbert-sequence-classification-with-out..> 31-Oct-2023 00:35               13454
-230-yolov8-instance-segmentation-with-output.rst   31-Oct-2023 00:35               58539
-230-yolov8-keypoint-detection-with-output.rst      31-Oct-2023 00:35               55065
-230-yolov8-object-detection-with-output.rst        31-Oct-2023 00:35               58903
-231-instruct-pix2pix-image-editing-with-output.rst 31-Oct-2023 00:35               52684
-232-clip-language-saliency-map-with-output.rst     31-Oct-2023 00:35               34908
-233-blip-convert-with-output.rst                   31-Oct-2023 00:35               29348
-233-blip-optimize-with-output.rst                  31-Oct-2023 00:35               17131
-234-encodec-audio-compression-with-output.rst      31-Oct-2023 00:35             3864226
-235-controlnet-stable-diffusion-with-output.rst    31-Oct-2023 00:35               58531
-236-stable-diffusion-v2-infinite-zoom-with-outp..> 31-Oct-2023 00:35               52546
-236-stable-diffusion-v2-optimum-demo-comparison..> 31-Oct-2023 00:35                8059
-236-stable-diffusion-v2-optimum-demo-with-outpu..> 31-Oct-2023 00:35                8407
-236-stable-diffusion-v2-text-to-image-demo-with..> 31-Oct-2023 00:35               13860
-236-stable-diffusion-v2-text-to-image-with-outp..> 31-Oct-2023 00:35               44177
-237-segment-anything-with-output.rst               31-Oct-2023 00:35               64142
-238-deep-floyd-if-optimize-with-output.rst         31-Oct-2023 00:35               31042
-239-image-bind-convert-with-output.rst             31-Oct-2023 00:35             2397351
-240-dolly-2-instruction-following-with-output.rst  31-Oct-2023 00:35               31788
-241-riffusion-text-to-music-with-output.rst        31-Oct-2023 00:35              623769
-242-freevc-voice-conversion-with-output.rst        31-Oct-2023 00:35              653238
-243-tflite-selfie-segmentation-with-output.rst     31-Oct-2023 00:35               22673
-244-named-entity-recognition-with-output.rst       31-Oct-2023 00:35               23869
-245-typo-detector-with-output.rst                  31-Oct-2023 00:35               30340
-246-depth-estimation-videpth-with-output.rst       31-Oct-2023 00:35               53677
-247-code-language-id-with-output.rst               31-Oct-2023 00:35               38729
-248-stable-diffusion-xl-with-output.rst            31-Oct-2023 00:35               23133
-249-oneformer-segmentation-with-output.rst         31-Oct-2023 00:35               28917
-250-music-generation-with-output.rst               31-Oct-2023 00:35             1390751
-251-tiny-sd-image-generation-with-output.rst       31-Oct-2023 00:35               47330
-252-fastcomposer-image-generation-with-output.rst  31-Oct-2023 00:35               46698
-253-zeroscope-text2video-with-output.rst           31-Oct-2023 00:35             1875395
-254-llm-chatbot-with-output.rst                    31-Oct-2023 00:35               43239
-255-mms-massively-multilingual-speech-with-outp..> 31-Oct-2023 00:35             1388124
-256-bark-text-to-audio-with-output.rst             31-Oct-2023 00:35              698945
-257-llava-multimodal-chatbot-with-output.rst       31-Oct-2023 00:35               52293
-258-blip-diffusion-subject-generation-with-outp..> 31-Oct-2023 00:35               52630
-259-decidiffusion-image-generation-with-output.rst 31-Oct-2023 00:35               51032
-260-pix2struct-docvqa-with-output.rst              31-Oct-2023 00:35               14557
-261-fast-segment-anything-with-output.rst          31-Oct-2023 00:35               22553
-262-softvc-voice-conversion-with-output.rst        31-Oct-2023 00:35               10075
-263-latent-consistency-models-image-generation-..> 31-Oct-2023 00:35               34965
-301-tensorflow-training-openvino-nncf-with-outp..> 31-Oct-2023 00:35               32073
-301-tensorflow-training-openvino-with-output.rst   31-Oct-2023 00:35               34000
-302-pytorch-quantization-aware-training-with-ou..> 31-Oct-2023 00:35               30565
-305-tensorflow-quantization-aware-training-with..> 31-Oct-2023 00:35               22473
-401-object-detection-with-output.rst               31-Oct-2023 00:35               20565
-402-pose-estimation-with-output.rst                31-Oct-2023 00:35               17581
-403-action-recognition-webcam-with-output.rst      31-Oct-2023 00:35               27886
-404-style-transfer-with-output.rst                 31-Oct-2023 00:35               17562
-405-paddle-ocr-webcam-with-output.rst              31-Oct-2023 00:35               26070
-406-3D-pose-estimation-with-output.rst             31-Oct-2023 00:35               33937
-407-person-tracking-with-output.rst                31-Oct-2023 00:35               28861
-notebook_utils-with-output.rst                     31-Oct-2023 00:35               12520
-notebooks_tags.json                                31-Oct-2023 00:35               10406
-notebooks_with_binder_buttons.txt                  31-Oct-2023 00:35                1166
-notebooks_with_colab_buttons.txt                   31-Oct-2023 00:35                1569
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/


../
+001-hello-world-with-output_files/                 07-Dec-2023 00:49                   -
+003-hello-segmentation-with-output_files/          07-Dec-2023 00:49                   -
+004-hello-detection-with-output_files/             07-Dec-2023 00:49                   -
+101-tensorflow-classification-to-openvino-with-..> 07-Dec-2023 00:49                   -
+102-pytorch-onnx-to-openvino-with-output_files/    07-Dec-2023 00:49                   -
+102-pytorch-to-openvino-with-output_files/         07-Dec-2023 00:49                   -
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49                   -
+106-auto-device-with-output_files/                 07-Dec-2023 00:49                   -
+109-latency-tricks-with-output_files/              07-Dec-2023 00:49                   -
+109-throughput-tricks-with-output_files/           07-Dec-2023 00:49                   -
+110-ct-scan-live-inference-with-output_files/      07-Dec-2023 00:49                   -
+110-ct-segmentation-quantize-nncf-with-output_f..> 07-Dec-2023 00:49                   -
+111-yolov5-quantization-migration-with-output_f..> 07-Dec-2023 00:49                   -
+113-image-classification-quantization-with-outp..> 07-Dec-2023 00:49                   -
+115-async-api-with-output_files/                   07-Dec-2023 00:49                   -
+117-model-server-with-output_files/                07-Dec-2023 00:49                   -
+118-optimize-preprocessing-with-output_files/      07-Dec-2023 00:49                   -
+119-tflite-to-openvino-with-output_files/          07-Dec-2023 00:49                   -
+120-tensorflow-instance-segmentation-to-openvin..> 07-Dec-2023 00:49                   -
+120-tensorflow-object-detection-to-openvino-wit..> 07-Dec-2023 00:49                   -
+123-detectron2-to-openvino-with-output_files/      07-Dec-2023 00:49                   -
+125-convnext-classification-with-output_files/     07-Dec-2023 00:49                   -
+125-lraspp-segmentation-with-output_files/         07-Dec-2023 00:49                   -
+126-tensorflow-hub-with-output_files/              07-Dec-2023 00:49                   -
+201-vision-monodepth-with-output_files/            07-Dec-2023 00:49                   -
+202-vision-superresolution-image-with-output_files 07-Dec-2023 00:49                   -
+203-meter-reader-with-output_files/                07-Dec-2023 00:49                   -
+204-segmenter-semantic-segmentation-with-output..> 07-Dec-2023 00:49                   -
+205-vision-background-removal-with-output_files/   07-Dec-2023 00:49                   -
+206-vision-paddlegan-anime-with-output_files/      07-Dec-2023 00:49                   -
+207-vision-paddlegan-superresolution-with-outpu..> 07-Dec-2023 00:49                   -
+208-optical-character-recognition-with-output_f..> 07-Dec-2023 00:49                   -
+209-handwritten-ocr-with-output_files/             07-Dec-2023 00:49                   -
+211-speech-to-text-with-output_files/              07-Dec-2023 00:49                   -
+212-pyannote-speaker-diarization-with-output_files 07-Dec-2023 00:49                   -
+215-image-inpainting-with-output_files/            07-Dec-2023 00:49                   -
+216-attention-center-with-output_files/            07-Dec-2023 00:49                   -
+217-vision-deblur-with-output_files/               07-Dec-2023 00:49                   -
+218-vehicle-detection-and-recognition-with-outp..> 07-Dec-2023 00:49                   -
+220-cross-lingual-books-alignment-with-output_f..> 07-Dec-2023 00:49                   -
+222-vision-image-colorization-with-output_files/   07-Dec-2023 00:49                   -
+224-3D-segmentation-point-clouds-with-output_files 07-Dec-2023 00:49                   -
+225-stable-diffusion-text-to-image-with-output_..> 07-Dec-2023 00:49                   -
+226-yolov7-optimization-with-output_files/         07-Dec-2023 00:49                   -
+228-clip-zero-shot-convert-with-output_files/      07-Dec-2023 00:49                   -
+228-clip-zero-shot-quantize-with-output_files/     07-Dec-2023 00:49                   -
+230-yolov8-instance-segmentation-with-output_files 07-Dec-2023 00:49                   -
+230-yolov8-keypoint-detection-with-output_files/   07-Dec-2023 00:49                   -
+230-yolov8-object-detection-with-output_files/     07-Dec-2023 00:49                   -
+231-instruct-pix2pix-image-editing-with-output_..> 07-Dec-2023 00:49                   -
+232-clip-language-saliency-map-with-output_files/  07-Dec-2023 00:49                   -
+233-blip-convert-with-output_files/                07-Dec-2023 00:49                   -
+233-blip-optimize-with-output_files/               07-Dec-2023 00:49                   -
+234-encodec-audio-compression-with-output_files/   07-Dec-2023 00:49                   -
+235-controlnet-stable-diffusion-with-output_files/ 07-Dec-2023 00:49                   -
+236-stable-diffusion-v2-optimum-demo-comparison..> 07-Dec-2023 00:49                   -
+236-stable-diffusion-v2-optimum-demo-with-outpu..> 07-Dec-2023 00:49                   -
+236-stable-diffusion-v2-text-to-image-demo-with..> 07-Dec-2023 00:49                   -
+237-segment-anything-with-output_files/            07-Dec-2023 00:49                   -
+238-deep-floyd-if-optimize-with-output_files/      07-Dec-2023 00:49                   -
+239-image-bind-convert-with-output_files/          07-Dec-2023 00:49                   -
+241-riffusion-text-to-music-with-output_files/     07-Dec-2023 00:49                   -
+243-tflite-selfie-segmentation-with-output_files/  07-Dec-2023 00:49                   -
+246-depth-estimation-videpth-with-output_files/    07-Dec-2023 00:49                   -
+248-ssd-b1-with-output_files/                      07-Dec-2023 00:49                   -
+248-stable-diffusion-xl-with-output_files/         07-Dec-2023 00:49                   -
+249-oneformer-segmentation-with-output_files/      07-Dec-2023 00:49                   -
+251-tiny-sd-image-generation-with-output_files/    07-Dec-2023 00:49                   -
+257-llava-multimodal-chatbot-with-output_files/    07-Dec-2023 00:49                   -
+258-blip-diffusion-subject-generation-with-outp..> 07-Dec-2023 00:49                   -
+259-decidiffusion-image-generation-with-output_..> 07-Dec-2023 00:49                   -
+260-pix2struct-docvqa-with-output_files/           07-Dec-2023 00:49                   -
+261-fast-segment-anything-with-output_files/       07-Dec-2023 00:49                   -
+263-latent-consistency-models-image-generation-..> 07-Dec-2023 00:49                   -
+263-lcm-lora-controlnet-with-output_files/         07-Dec-2023 00:49                   -
+265-wuerstchen-image-generation-with-output_files/ 07-Dec-2023 00:49                   -
+269-film-slowmo-with-output_files/                 07-Dec-2023 00:49                   -
+271-sdxl-turbo-with-output_files/                  07-Dec-2023 00:49                   -
+301-tensorflow-training-openvino-nncf-with-outp..> 07-Dec-2023 00:49                   -
+301-tensorflow-training-openvino-with-output_files 07-Dec-2023 00:49                   -
+305-tensorflow-quantization-aware-training-with..> 07-Dec-2023 00:49                   -
+401-object-detection-with-output_files/            07-Dec-2023 00:49                   -
+402-pose-estimation-with-output_files/             07-Dec-2023 00:49                   -
+403-action-recognition-webcam-with-output_files/   07-Dec-2023 00:49                   -
+404-style-transfer-with-output_files/              07-Dec-2023 00:49                   -
+405-paddle-ocr-webcam-with-output_files/           07-Dec-2023 00:49                   -
+407-person-tracking-with-output_files/             07-Dec-2023 00:49                   -
+notebook_utils-with-output_files/                  07-Dec-2023 00:49                   -
+001-hello-world-with-output.rst                    07-Dec-2023 00:49                5254
+002-openvino-api-with-output.rst                   07-Dec-2023 00:49               38674
+003-hello-segmentation-with-output.rst             07-Dec-2023 00:49                6880
+004-hello-detection-with-output.rst                07-Dec-2023 00:49                8034
+101-tensorflow-classification-to-openvino-with-..> 07-Dec-2023 00:49               11407
+102-pytorch-onnx-to-openvino-with-output.rst       07-Dec-2023 00:49               19675
+102-pytorch-to-openvino-with-output.rst            07-Dec-2023 00:49               24458
+103-paddle-to-openvino-classification-with-outp..> 07-Dec-2023 00:49               17004
+104-model-tools-with-output.rst                    07-Dec-2023 00:49               21016
+105-language-quantize-bert-with-output.rst         07-Dec-2023 00:49               20138
+106-auto-device-with-output.rst                    07-Dec-2023 00:49               24070
+107-speech-recognition-quantization-data2vec-wi..> 07-Dec-2023 00:49              970151
+107-speech-recognition-quantization-wav2vec2-wi..> 07-Dec-2023 00:49              223348
+108-gpu-device-with-output.rst                     07-Dec-2023 00:49               55430
+109-latency-tricks-with-output.rst                 07-Dec-2023 00:49               24164
+109-throughput-tricks-with-output.rst              07-Dec-2023 00:49               26370
+110-ct-scan-live-inference-with-output.rst         07-Dec-2023 00:49               17892
+110-ct-segmentation-quantize-nncf-with-output.rst  07-Dec-2023 00:49               38096
+111-yolov5-quantization-migration-with-output.rst  07-Dec-2023 00:49               50502
+112-pytorch-post-training-quantization-nncf-wit..> 07-Dec-2023 00:49               29261
+113-image-classification-quantization-with-outp..> 07-Dec-2023 00:49               22890
+115-async-api-with-output.rst                      07-Dec-2023 00:49               19965
+116-sparsity-optimization-with-output.rst          07-Dec-2023 00:49               17816
+117-model-server-with-output.rst                   07-Dec-2023 00:49               21790
+118-optimize-preprocessing-with-output.rst         07-Dec-2023 00:49               21592
+119-tflite-to-openvino-with-output.rst             07-Dec-2023 00:49               11397
+120-tensorflow-instance-segmentation-to-openvin..> 07-Dec-2023 00:49               25278
+120-tensorflow-object-detection-to-openvino-wit..> 07-Dec-2023 00:49               27517
+121-convert-to-openvino-with-output.rst            07-Dec-2023 00:49               90848
+122-speech-recognition-quantization-wav2vec2-wi..> 07-Dec-2023 00:49              483195
+122-yolov8-quantization-with-accuracy-control-w..> 07-Dec-2023 00:49               22498
+123-detectron2-to-openvino-with-output.rst         07-Dec-2023 00:49               16684
+124-hugging-face-hub-with-output.rst               07-Dec-2023 00:49               24923
+125-convnext-classification-with-output.rst        07-Dec-2023 00:49               11040
+125-lraspp-segmentation-with-output.rst            07-Dec-2023 00:49               11034
+126-tensorflow-hub-with-output.rst                 07-Dec-2023 00:49               15099
+201-vision-monodepth-with-output.rst               07-Dec-2023 00:49              969377
+202-vision-superresolution-image-with-output.rst   07-Dec-2023 00:49               27153
+202-vision-superresolution-video-with-output.rst   07-Dec-2023 00:49             9641843
+203-meter-reader-with-output.rst                   07-Dec-2023 00:49               26832
+204-segmenter-semantic-segmentation-with-output..> 07-Dec-2023 00:49               27572
+205-vision-background-removal-with-output.rst      07-Dec-2023 00:49               14003
+206-vision-paddlegan-anime-with-output.rst         07-Dec-2023 00:49               21580
+207-vision-paddlegan-superresolution-with-outpu..> 07-Dec-2023 00:49               17587
+208-optical-character-recognition-with-output.rst  07-Dec-2023 00:49               27964
+209-handwritten-ocr-with-output.rst                07-Dec-2023 00:49               12725
+210-slowfast-video-recognition-with-output.rst     07-Dec-2023 00:49              768967
+211-speech-to-text-with-output.rst                 07-Dec-2023 00:49               88279
+212-pyannote-speaker-diarization-with-output.rst   07-Dec-2023 00:49             1294686
+213-question-answering-with-output.rst             07-Dec-2023 00:49               22507
+214-grammar-correction-with-output.rst             07-Dec-2023 00:49               31853
+215-image-inpainting-with-output.rst               07-Dec-2023 00:49                9781
+216-attention-center-with-output.rst               07-Dec-2023 00:49               14112
+217-vision-deblur-with-output.rst                  07-Dec-2023 00:49               11904
+218-vehicle-detection-and-recognition-with-outp..> 07-Dec-2023 00:49               17774
+219-knowledge-graphs-conve-with-output.rst         07-Dec-2023 00:49               25536
+220-cross-lingual-books-alignment-with-output.rst  07-Dec-2023 00:49               50005
+221-machine-translation-with-output.rst            07-Dec-2023 00:49                9859
+222-vision-image-colorization-with-output.rst      07-Dec-2023 00:49               19076
+223-text-prediction-with-output.rst                07-Dec-2023 00:49               27142
+224-3D-segmentation-point-clouds-with-output.rst   07-Dec-2023 00:49               10121
+225-stable-diffusion-text-to-image-with-output.rst 07-Dec-2023 00:49               55735
+226-yolov7-optimization-with-output.rst            07-Dec-2023 00:49               46096
+227-whisper-convert-with-output.rst                07-Dec-2023 00:49               17645
+227-whisper-nncf-quantize-with-output.rst          07-Dec-2023 00:49               23414
+228-clip-zero-shot-convert-with-output.rst         07-Dec-2023 00:49               14962
+228-clip-zero-shot-quantize-with-output.rst        07-Dec-2023 00:49               15461
+229-distilbert-sequence-classification-with-out..> 07-Dec-2023 00:49               13235
+230-yolov8-instance-segmentation-with-output.rst   07-Dec-2023 00:49               57650
+230-yolov8-keypoint-detection-with-output.rst      07-Dec-2023 00:49               54173
+230-yolov8-object-detection-with-output.rst        07-Dec-2023 00:49               57728
+231-instruct-pix2pix-image-editing-with-output.rst 07-Dec-2023 00:49               52168
+232-clip-language-saliency-map-with-output.rst     07-Dec-2023 00:49               34489
+233-blip-convert-with-output.rst                   07-Dec-2023 00:49               28629
+233-blip-optimize-with-output.rst                  07-Dec-2023 00:49               16575
+234-encodec-audio-compression-with-output.rst      07-Dec-2023 00:49             3863698
+235-controlnet-stable-diffusion-with-output.rst    07-Dec-2023 00:49               55588
+236-stable-diffusion-v2-infinite-zoom-with-outp..> 07-Dec-2023 00:49               52084
+236-stable-diffusion-v2-optimum-demo-comparison..> 07-Dec-2023 00:49                7935
+236-stable-diffusion-v2-optimum-demo-with-outpu..> 07-Dec-2023 00:49                8283
+236-stable-diffusion-v2-text-to-image-demo-with..> 07-Dec-2023 00:49               13530
+236-stable-diffusion-v2-text-to-image-with-outp..> 07-Dec-2023 00:49               43738
+237-segment-anything-with-output.rst               07-Dec-2023 00:49               62765
+238-deep-floyd-if-optimize-with-output.rst         07-Dec-2023 00:49               30528
+239-image-bind-convert-with-output.rst             07-Dec-2023 00:49             2396943
+240-dolly-2-instruction-following-with-output.rst  07-Dec-2023 00:49               30765
+241-riffusion-text-to-music-with-output.rst        07-Dec-2023 00:49              623693
+243-tflite-selfie-segmentation-with-output.rst     07-Dec-2023 00:49               22157
+244-named-entity-recognition-with-output.rst       07-Dec-2023 00:49               20326
+245-typo-detector-with-output.rst                  07-Dec-2023 00:49               28788
+246-depth-estimation-videpth-with-output.rst       07-Dec-2023 00:49               53035
+247-code-language-id-with-output.rst               07-Dec-2023 00:49               36862
+248-ssd-b1-with-output.rst                         07-Dec-2023 00:49               18947
+248-stable-diffusion-xl-with-output.rst            07-Dec-2023 00:49               22703
+249-oneformer-segmentation-with-output.rst         07-Dec-2023 00:49               28293
+250-music-generation-with-output.rst               07-Dec-2023 00:49             1393573
+251-tiny-sd-image-generation-with-output.rst       07-Dec-2023 00:49               46798
+252-fastcomposer-image-generation-with-output.rst  07-Dec-2023 00:49               46119
+253-zeroscope-text2video-with-output.rst           07-Dec-2023 00:49             1876054
+254-llm-chatbot-with-output.rst                    07-Dec-2023 00:49               48901
+255-mms-massively-multilingual-speech-with-outp..> 07-Dec-2023 00:49             1390002
+256-bark-text-to-audio-with-output.rst             07-Dec-2023 00:49              698540
+257-llava-multimodal-chatbot-with-output.rst       07-Dec-2023 00:49               55772
+258-blip-diffusion-subject-generation-with-outp..> 07-Dec-2023 00:49               53393
+259-decidiffusion-image-generation-with-output.rst 07-Dec-2023 00:49               50338
+260-pix2struct-docvqa-with-output.rst              07-Dec-2023 00:49               14327
+261-fast-segment-anything-with-output.rst          07-Dec-2023 00:49               36293
+262-softvc-voice-conversion-with-output.rst        07-Dec-2023 00:49               10132
+263-latent-consistency-models-image-generation-..> 07-Dec-2023 00:49               45989
+263-lcm-lora-controlnet-with-output.rst            07-Dec-2023 00:49               66944
+264-qrcode-monster-with-output.rst                 07-Dec-2023 00:49               40878
+265-wuerstchen-image-generation-with-output.rst    07-Dec-2023 00:49               15781
+266-speculative-sampling-with-output.rst           07-Dec-2023 00:49               14848
+267-distil-whisper-asr-with-output.rst             07-Dec-2023 00:49             3460038
+268-table-question-answering-with-output.rst       07-Dec-2023 00:49               28733
+269-film-slowmo-with-output.rst                    07-Dec-2023 00:49             2090052
+270-sound-generation-audioldm2-with-output.rst     07-Dec-2023 00:49              367989
+271-sdxl-turbo-with-output.rst                     07-Dec-2023 00:49               31083
+272-paint-by-example-with-output.rst               07-Dec-2023 00:49               52442
+301-tensorflow-training-openvino-nncf-with-outp..> 07-Dec-2023 00:49               53590
+301-tensorflow-training-openvino-with-output.rst   07-Dec-2023 00:49               44409
+302-pytorch-quantization-aware-training-with-ou..> 07-Dec-2023 00:49               30524
+305-tensorflow-quantization-aware-training-with..> 07-Dec-2023 00:49               24552
+401-object-detection-with-output.rst               07-Dec-2023 00:49               19275
+402-pose-estimation-with-output.rst                07-Dec-2023 00:49               17019
+403-action-recognition-webcam-with-output.rst      07-Dec-2023 00:49               27306
+404-style-transfer-with-output.rst                 07-Dec-2023 00:49               16956
+405-paddle-ocr-webcam-with-output.rst              07-Dec-2023 00:49               25566
+406-3D-pose-estimation-with-output.rst             07-Dec-2023 00:49               33773
+407-person-tracking-with-output.rst                07-Dec-2023 00:49               28265
+all_notebooks_paths.txt                            07-Dec-2023 00:49                9708
+notebook_utils-with-output.rst                     07-Dec-2023 00:49               13124
+notebooks_tags.json                                07-Dec-2023 00:49               11338
+notebooks_with_binder_buttons.txt                  07-Dec-2023 00:49                1166
+notebooks_with_colab_buttons.txt                   07-Dec-2023 00:49                1598
 

diff --git a/docs/notebooks/notebook_utils-with-output.rst b/docs/notebooks/notebook_utils-with-output.rst index 7d23c0ccae3..e4311baee76 100644 --- a/docs/notebooks/notebook_utils-with-output.rst +++ b/docs/notebooks/notebook_utils-with-output.rst @@ -5,13 +5,12 @@ This notebook contains helper functions and classes for use with OpenVINO™ Notebooks. The code is synchronized with the ``notebook_utils.py`` file in the same directory as this notebook. -There are five categories: +There are four categories: - `Files <#files>`__ - `Images <#images>`__ - `Videos <#videos>`__ - `Visualization <#visualization>`__ -- `OpenVINO Tools <#openvino-tools>`__ - `Checks and Alerts <#checks-and-alerts>`__ Each category contains a test cell that also shows how to use the @@ -33,6 +32,8 @@ functions in the section. Files ----- + + Load an image, download a file, download an OpenVINO IR model, and create a progress bar to show download progress. @@ -58,6 +59,8 @@ create a progress bar to show download progress. Test File Functions ~~~~~~~~~~~~~~~~~~~ + + .. code:: ipython3 model_url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/segmentation.xml" @@ -91,7 +94,7 @@ Test File Functions .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-534/.workspace/scm/ov-notebook/notebooks/utils/Safety_Full_Hat_and_Vest.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-561/.workspace/scm/ov-notebook/notebooks/utils/Safety_Full_Hat_and_Vest.mp4 .. code:: ipython3 @@ -108,12 +111,12 @@ Test File Functions .. parsed-literal:: - openvino_notebooks_readme.md: 0%| | 0.00/13.8k [00:00 -Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/notebook_utils-with-output_files/ +Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/notebook_utils-with-output_files/ -

Index of /projects/ov-notebook/0.1.0-latest/20231030220807/dist/rst_files/notebook_utils-with-output_files/


../
-notebook_utils-with-output_12_0.jpg                31-Oct-2023 00:35              121563
-notebook_utils-with-output_12_0.png                31-Oct-2023 00:35              869307
-notebook_utils-with-output_26_0.png                31-Oct-2023 00:35               45498
-notebook_utils-with-output_41_0.png                31-Oct-2023 00:35               10059
-notebook_utils-with-output_41_1.png                31-Oct-2023 00:35               37584
-notebook_utils-with-output_41_2.png                31-Oct-2023 00:35               16690
-notebook_utils-with-output_41_3.png                31-Oct-2023 00:35               38992
+

Index of /projects/ov-notebook/0.1.0-latest/20231206220809/dist/rst_files/notebook_utils-with-output_files/


../
+notebook_utils-with-output_12_0.jpg                07-Dec-2023 00:49              121563
+notebook_utils-with-output_12_0.png                07-Dec-2023 00:49              869307
+notebook_utils-with-output_26_0.png                07-Dec-2023 00:49               43773
+notebook_utils-with-output_41_0.png                07-Dec-2023 00:49               10059
+notebook_utils-with-output_41_1.png                07-Dec-2023 00:49               37584
+notebook_utils-with-output_41_2.png                07-Dec-2023 00:49               16690
+notebook_utils-with-output_41_3.png                07-Dec-2023 00:49               38992
 

diff --git a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_26_0.png b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_26_0.png index bfbb21457a5..d6b5e4a29bf 100644 --- a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_26_0.png +++ b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_26_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a447faf6463b3efa6f6d4c8bcedd6731e29c3807e9aa8dff94973e5e6c0263f5 -size 45498 +oid sha256:0b1d0537d8cbfb960b949e320a014f87a4c5d8b6db7b1714bdd0603f561e2503 +size 43773 diff --git a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_0.png b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_0.png index 6395be6a3a3..ce444cbd6d2 100644 --- a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_0.png +++ b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5b51a1aaed162de717fc3e4d8c60add72da2c425c61a1d0c78b81284306559c7 +oid sha256:2f7387728cbddbbf39ef89389cb6bf5f4f3629b61c3c0a46079e6458dd969fc7 size 10059 diff --git a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_1.png b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_1.png index 8293bf48530..4366e20c569 100644 --- a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_1.png +++ b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f5bf5d56666a54aefb7230259cc93c7ab32b775d19c70a37c3f057f002e9780 +oid sha256:1977fd5fca0211a53997161d8532792b4f4477ba74026a2cf706dc4d073d440f size 37584 diff --git a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_2.png b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_2.png index 117bfd26781..4d411422f3c 100644 --- a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_2.png +++ b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_2.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8cd96d054ba6bbe07433f6d913e7a2113cb19b689403ed8bdd1e632088a9e029 +oid sha256:7e626d98855cd395c6a5fa82e32e40ccb77fa5e3763cc4952fffe288dee82678 size 16690 diff --git a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_3.png b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_3.png index b715839e69d..4ac8d5477bc 100644 --- a/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_3.png +++ b/docs/notebooks/notebook_utils-with-output_files/notebook_utils-with-output_41_3.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd7d398d3061ea2c89b816639925d342909012cb03786d00bef4100943ea24fa +oid sha256:eee1b0a122156df270e09ae5a6b2749f6c6182a1fcd31bdf00c62dafcdf7c941 size 38992 diff --git a/docs/notebooks/notebooks_tags.json b/docs/notebooks/notebooks_tags.json index 3979ba34021..7cf917b57cf 100644 --- a/docs/notebooks/notebooks_tags.json +++ b/docs/notebooks/notebooks_tags.json @@ -1,7 +1,9 @@ { "002-openvino-api": [ "ONNX", - "Reshape Model" + "Pytorch", + "Reshape Model", + "Torchvision" ], "101-tensorflow-classification-to-openvino": [ "Tensorflow" @@ -389,7 +391,6 @@ "Async Inference", "NNCF", "Pytorch", - "Reshape Model", "Transformers" ], "255-mms-massively-multilingual-speech": [ @@ -433,6 +434,12 @@ "Pytorch", "Transformers" ], + "263-lcm-lora-controlnet": [ + "Benchmark Model", + "NNCF", + "Pytorch", + "Transformers" + ], "264-qrcode-monster": [ "Pytorch", "Transformers" @@ -456,6 +463,19 @@ "269-film-slowmo": [ "Tensorflow" ], + "270-sound-generation-audioldm2": [ + "Pytorch" + ], + "271-sdxl-turbo": [ + "Benchmark Model", + "NNCF", + "Pytorch", + "Transformers" + ], + "272-paint-by-example": [ + "Pytorch", + "Transformers" + ], "301-tensorflow-training-openvino-nncf": [ "Benchmark Model", "NNCF",