From 6cd70af7c201d5a24bc6b4a965c45156146e9cdb Mon Sep 17 00:00:00 2001 From: Sebastian Golebiewski Date: Tue, 22 Aug 2023 15:37:37 +0200 Subject: [PATCH] update-notebooks (#19337) --- .../250-music-generation-with-output.rst | 574 +++++++++ ...1-tiny-sd-image-generation-with-output.rst | 1136 +++++++++++++++++ .../251-tiny-sd-image-generation_33_1.png | 3 + .../251-tiny-sd-image-generation_37_1.png | 3 + .../251-tiny-sd-image-generation_39_1.png | 3 + .../index.html | 0 .../notebooks_with_binder_buttons.txt | 1 + .../notebooks_with_colab_buttons.txt | 2 + docs/tutorials.md | 24 +- 9 files changed, 1743 insertions(+), 3 deletions(-) create mode 100644 docs/notebooks/250-music-generation-with-output.rst create mode 100644 docs/notebooks/251-tiny-sd-image-generation-with-output.rst create mode 100644 docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_33_1.png create mode 100644 docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_37_1.png create mode 100644 docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_39_1.png create mode 100644 docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html diff --git a/docs/notebooks/250-music-generation-with-output.rst b/docs/notebooks/250-music-generation-with-output.rst new file mode 100644 index 00000000000..1339c538e7e --- /dev/null +++ b/docs/notebooks/250-music-generation-with-output.rst @@ -0,0 +1,574 @@ +Controllable Music Generation with MusicGen and OpenVINO +======================================================== + +.. _top: + +MusicGen is a single-stage auto-regressive Transformer model capable of +generating high-quality music samples conditioned on text descriptions +or audio prompts. The text prompt is passed to a text encoder model (T5) +to obtain a sequence of hidden-state representations. These hidden +states are fed to MusicGen, which predicts discrete audio tokens (audio +codes). Finally, audio tokens are then decoded using an audio +compression model (EnCodec) to recover the audio waveform. + +.. figure:: https://user-images.githubusercontent.com/76463150/260439306-81c81c8d-1f9c-41d0-b881-9491766def8e.png + :alt: pipeline + + pipeline + +`The MusicGen model `__ does not +require a self-supervised semantic representation of the text/audio +prompts; it operates over several streams of compressed discrete music +representation with efficient token interleaving patterns, thus +eliminating the need to cascade multiple models to predict a set of +codebooks (e.g. hierarchically or upsampling). Unlike prior models +addressing music generation, it is able to generate all the codebooks in +a single forward pass. + +In this tutorial, we consider how to run the MusicGen model using +OpenVINO. + +We will use a model implementation from the `Hugging Face +Transformers `__ +library. + +**Table of contents**: + +- `Requirements and Imports <#prerequisites>`__ +- `Original Pipeline Inference <#musicgen-in-hf-transformers>`__ +- `Converting the Models to OpenVINO IR <#convert-models-to-openvino-intermediate-representation-ir-format>`__ +- `Convert Text Encoder <#convert-text-encoder>`__ +- `Convert MusicGen Language Model <#convert-musicgen-language-model>`__ +- `Convert Audio Decoder <#convert-audio-decoder>`__ +- `Embedding the Converted Models into the Pipeline <#embedding-the-converted-models-into-the-original-pipeline>`__ +- `Run Gradio App <#try-out-the-converted-pipeline>`__ + +Prerequisites `⇑ <#top>`__ +############################################################################################################################### + +Install requirements `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. code:: ipython3 + + !pip install -q "openvino==2023.1.0.dev20230811" + !pip install -q torch onnx gradio + !pip install -q transformers + +Imports `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. code:: ipython3 + + from collections import namedtuple + import gc + from pathlib import Path + from typing import Optional, Tuple + import warnings + + from IPython.display import Audio + from openvino import Core, convert_model, PartialShape, save_model, Type + import numpy as np + import torch + from torch.jit import TracerWarning + from transformers import AutoProcessor, MusicgenForConditionalGeneration + from transformers.modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions + + # Ignore tracing warnings + warnings.filterwarnings("ignore", category=TracerWarning) + +MusicGen in HF Transformers `⇑ <#top>`__ +############################################################################################################################### + +To work with +`MusicGen `__ by Meta +AI, we will use `Hugging Face Transformers +package `__. Transformers +package exposes the ``MusicgenForConditionalGeneration`` class, +simplifying the model instantiation and weights loading. The code below +demonstrates how to create a ``MusicgenForConditionalGeneration`` and +generate a text-conditioned music sample. + +.. code:: ipython3 + + # Load the pipeline + model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small", torchscript=True, return_dict=False) + +In the cell below user is free to change PyTorch model inference device +and the desired music sample length. + +.. code:: ipython3 + + device = "cpu" + sample_length = 8 # seconds + + n_tokens = sample_length * model.config.audio_encoder.frame_rate + 3 + sampling_rate = model.config.audio_encoder.sampling_rate + print('Sampling rate is', sampling_rate, 'Hz') + + model.to(device) + model.eval(); + + +.. parsed-literal:: + + Sampling rate is 32000 Hz + + +Original Pipeline Inference `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Text Preprocessing prepares the text prompt to be fed into the model, +the ``processor`` object abstracts this step for us. Text tokenization +is performed under the hood, it assigning tokens or IDs to the words; in +other words, token IDs are just indices of the words in the model +vocabulary. It helps the model understand the context of a sentence. + +.. code:: ipython3 + + processor = AutoProcessor.from_pretrained("facebook/musicgen-small") + + inputs = processor( + text=["80s pop track with bassy drums and synth"], + return_tensors="pt", + ) + + audio_values = model.generate(**inputs.to(device), do_sample=True, guidance_scale=3, max_new_tokens=n_tokens) + + Audio(audio_values[0].cpu().numpy(), rate=sampling_rate) + + + + +.. raw:: html + + + + + + + +Convert models to OpenVINO Intermediate representation (IR) format `⇑ <#top>`__ +############################################################################################################################### + +Model conversion API enables direct conversion of PyTorch models. We +will utilize the ``openvino.convert_model`` method to acquire OpenVINO +IR versions of the models. The method requires a model object and +example input for model tracing. Under the hood, the converter will use +the PyTorch JIT compiler, to build a frozen model graph. + +The pipeline consists of three important parts: + +- The `T5 text encoder `__ + that translates user prompts into vectors in the latent space that + the next model - the MusicGen decoder can utilize. +- The `MusicGen Language + Model `__ + that auto-regressively generates audio tokens (codes). +- The `EnCodec model `__ + (we will use only the decoder part of it) is used to decode the audio + waveform from the audio tokens predicted by the MusicGen Language + Model. + +Let us convert each model step by step. + +0. Set Up Variables `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. code:: ipython3 + + models_dir = Path("./models") + t5_ir_path = models_dir / "t5.xml" + musicgen_0_ir_path = models_dir / "mg_0.xml" + musicgen_ir_path = models_dir / "mg.xml" + audio_decoder_onnx_path = models_dir / "encodec.onnx" + audio_decoder_ir_path = models_dir / "encodec.xml" + +1. Convert Text Encoder `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +The text encoder is responsible for converting the input prompt, such as +“90s rock song with loud guitars and heavy drums” into an embedding +space that can be fed to the next model. Typically, it is a +transformer-based encoder that maps a sequence of input tokens to a +sequence of text embeddings. + +The input for the text encoder consists of a tensor ``input_ids``, which +contains token indices from the text processed by the tokenizer and +``attention_mask`` that we will ignore as we will process one prompt at +a time and this vector will just consist of ones. + +We use OpenVINO Converter (OVC) below to convert the PyTorch model to +the OpenVINO Intermediate Representation format (IR), which you can +infer later with `OpenVINO +runtime `__ + +.. code:: ipython3 + + if not t5_ir_path.exists(): + t5_ov = convert_model(model.text_encoder, example_input={'input_ids': inputs['input_ids']}) + + save_model(t5_ov, t5_ir_path) + del t5_ov + gc.collect() + +2. Convert MusicGen Language Model `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +This model is the central part of the whole pipeline, it takes the +embedded text representation and generates audio codes that can be then +decoded into actual music. The model outputs several streams of audio +codes - tokens sampled from the pre-trained codebooks representing music +efficiently with a lower frame rate. The model employs innovative codes +intervaling strategy, that makes single-stage generation possible. + +On the 0th generation step the model accepts ``input_ids`` representing +the indices of audio codes, ``encoder_hidden_states`` and +``encoder_attention_mask`` that were provided by the text encoder. + +.. code:: ipython3 + + # Set model config `torchscript` to True, so the model returns a tuple as output + model.decoder.config.torchscript = True + + if not musicgen_0_ir_path.exists(): + decoder_input = { + 'input_ids': torch.ones(8, 1, dtype=torch.int64), + 'encoder_hidden_states': torch.ones(2, 12, 1024, dtype=torch.float32), + 'encoder_attention_mask': torch.ones(2, 12, dtype=torch.int64), + } + mg_ov_0_step = convert_model(model.decoder, example_input=decoder_input) + + save_model(mg_ov_0_step, musicgen_0_ir_path) + del mg_ov_0_step + gc.collect() + +On further iterations, the model is also provided with a +``past_key_values`` argument that contains previous outputs of the +attention block, it allows us to save on computations. But for us, it +means that the signature of the model’s ``forward`` method changed. +Models in OpenVINO IR have frozen calculation graphs and do not allow +optional arguments, that is why the MusicGen model must be converted a +second time, with an increased number of inputs. + +.. code:: ipython3 + + # Add additional argument to the example_input dict + if not musicgen_ir_path.exists(): + # Add `past_key_values` to the converted model signature + decoder_input['past_key_values'] = tuple( + [( + torch.ones(2, 16, 1, 64, dtype=torch.float32), + torch.ones(2, 16, 1, 64, dtype=torch.float32), + torch.ones(2, 16, 12, 64, dtype=torch.float32), + torch.ones(2, 16, 12, 64, dtype=torch.float32), + )] * 24 + ) + + mg_ov = convert_model(model.decoder, example_input=decoder_input) + +Moreover, the ``past_key_values`` argument is passed as +``Tuple[Tuple[torch.tensor]]`` which is a hard call for the converter. +The converter flattens these tuples, but have hard time identifying the +correct shapes for the tensors. The code below goes over raveled +``past_key_values``-related model inputs and sets the correct shape and +type for them. + +.. code:: ipython3 + + if not musicgen_ir_path.exists(): + for input in mg_ov.inputs[3:]: + input.get_node().set_partial_shape(PartialShape([-1, 16, -1, 64])) + input.get_node().set_element_type(Type.f32) + + mg_ov.validate_nodes_and_infer_types() + + save_model(mg_ov, musicgen_ir_path) + del mg_ov + gc.collect() + +3. Convert Audio Decoder `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +The audio decoder which is a part of the EnCodec model is used to +recover the audio waveform from the audio tokens predicted by the +MusicGen decoder. To learn more about the model please refer to the +corresponding `OpenVINO +example `__. + +The audio decoder computation graph contains an LSTM sub-network which +currently should be converted through ONNX. To do this we create a +wrapper class with its ``forward()`` method calling +``encodec.decode()``. + +.. code:: ipython3 + + if not audio_decoder_onnx_path.exists(): + class AudioDecoder(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, output_ids): + return self.model.decode(output_ids, [None]) + + audio_decoder_input = {'output_ids': torch.ones(1, 1, 4, n_tokens - 3, dtype=torch.int64),} + + with torch.no_grad(): + torch.onnx.export( + model=AudioDecoder(model.audio_encoder), + args=audio_decoder_input, + f=audio_decoder_onnx_path, + input_names=['output_ids',], + output_names=['decoded_audio'], + dynamic_axes={ + 'output_ids': {3: 'sequence_length'}, + 'decoded_audio': {2: 'audio_values'} + } + ) + + +.. parsed-literal:: + + /home/idavidyu/.virtualenvs/test/lib/python3.10/site-packages/torch/onnx/symbolic_opset9.py:4476: UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable length with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model. + warnings.warn( + + +.. parsed-literal:: + + ============= Diagnostic Run torch.onnx.export version 2.0.1+cu117 ============= + verbose: False, log level: Level.ERROR + ======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ======================== + + + +Now we can convert the frozen ONNX computation graph to OpenVINO IR. + +.. code:: ipython3 + + # Now we can convert the model to OpenVINO IR + if not audio_decoder_ir_path.exists(): + audio_decoder_ov = convert_model(str(audio_decoder_onnx_path)) + + save_model(audio_decoder_ov, audio_decoder_ir_path) + del audio_decoder_ov + gc.collect() + +Embedding the converted models into the original pipeline `⇑ <#top>`__ +############################################################################################################################### + +OpenVINO™ Runtime Python API is used to compile the model in OpenVINO IR +format. The +`Core `__ +class provides access to the OpenVINO Runtime API. The ``core`` object, +which is an instance of the ``Core`` class represents the API and it is +used to compile the model. + +.. code:: ipython3 + + core = Core() + +Select inference device `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Select device that will be used to do models inference using OpenVINO +from the dropdown list: + +.. code:: ipython3 + + import ipywidgets as widgets + + DEVICE = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + DEVICE + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=2, options=('CPU', 'GPU', 'AUTO'), value='AUTO') + + + +Adapt OpenVINO models to the original pipeline `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Here we create wrapper classes for all three OpenVINO models that we +want to embed in the original inference pipeline. Here are some of the +things to consider when adapting an OV model: - Make sure that +parameters passed by the original pipeline are forwarded to the compiled +OV model properly; sometimes the OV model uses only a portion of the +input arguments and some are ignored, sometimes you need to convert the +argument to another data type or unwrap some data structures such as +tuples or dictionaries. - Guarantee that the wrapper class returns +results to the pipeline in an expected format. In the example below you +can see how we pack OV model outputs into special classes declared in +the HF repo. - Pay attention to the model method used in the original +pipeline for calling the model - it may be not the ``forward`` method! +Refer to the ``AudioDecoderWrapper`` to see how we wrap OV model +inference into the ``decode`` method. + +.. code:: ipython3 + + class TextEncoderWrapper(torch.nn.Module): + def __init__(self, encoder_ir, config): + super().__init__() + self.encoder = core.compile_model(encoder_ir, DEVICE.value) + self.config = config + + def forward(self, input_ids, **kwargs): + last_hidden_state = self.encoder(input_ids)[self.encoder.outputs[0]] + last_hidden_state = torch.tensor(last_hidden_state) + return BaseModelOutputWithPastAndCrossAttentions(last_hidden_state=last_hidden_state) + + + class MusicGenWrapper(torch.nn.Module): + def __init__(self, music_gen_lm_0_ir, music_gen_lm_ir, config, num_codebooks, build_delay_pattern_mask, + apply_delay_pattern_mask): + super().__init__() + self.music_gen_lm_0 = core.compile_model(music_gen_lm_0_ir, DEVICE.value) + self.music_gen_lm = core.compile_model(music_gen_lm_ir, DEVICE.value) + self.config = config + self.num_codebooks = num_codebooks + self.build_delay_pattern_mask = build_delay_pattern_mask + self.apply_delay_pattern_mask = apply_delay_pattern_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + encoder_hidden_states: torch.FloatTensor = None, + encoder_attention_mask: torch.LongTensor = None, + past_key_values: Optional[Tuple[torch.FloatTensor]] = None, + **kwargs + ): + if past_key_values is None: + model = self.music_gen_lm_0 + arguments = (input_ids, encoder_hidden_states, encoder_attention_mask) + else: + model = self.music_gen_lm + arguments = (input_ids, encoder_hidden_states, encoder_attention_mask, *past_key_values) + + output = model(arguments) + return CausalLMOutputWithCrossAttentions( + logits=torch.tensor(output[model.outputs[0]]), + past_key_values=tuple([output[model.outputs[i]] for i in range(1, 97)]), + ) + + class AudioDecoderWrapper(torch.nn.Module): + def __init__(self, decoder_ir, config): + super().__init__() + self.decoder = core.compile_model(decoder_ir, DEVICE.value) + self.config = config + self.output_type = namedtuple("AudioDecoderOutput", ["audio_values"]) + + def decode(self, output_ids, audio_scales): + output = self.decoder(output_ids)[self.decoder.outputs[0]] + return self.output_type(audio_values=torch.tensor(output)) + +Now we initialize the wrapper objects and load them to the HF pipeline + +.. code:: ipython3 + + text_encode_ov = TextEncoderWrapper(t5_ir_path, model.text_encoder.config) + musicgen_decoder_ov = MusicGenWrapper( + musicgen_0_ir_path, + musicgen_ir_path, + model.decoder.config, + model.decoder.num_codebooks, + model.decoder.build_delay_pattern_mask, + model.decoder.apply_delay_pattern_mask + ) + audio_encoder_ov = AudioDecoderWrapper(audio_decoder_ir_path, model.audio_encoder.config) + + del model.text_encoder + del model.decoder + del model.audio_encoder + gc.collect() + + model.text_encoder = text_encode_ov + model.decoder = musicgen_decoder_ov + model.audio_encoder = audio_encoder_ov + +We can now infer the pipeline backed by OpenVINO models. + +.. code:: ipython3 + + processor = AutoProcessor.from_pretrained("facebook/musicgen-small") + + inputs = processor( + text=["80s pop track with bassy drums and synth"], + return_tensors="pt", + ) + + audio_values = model.generate(**inputs.to(device), do_sample=True, guidance_scale=3, max_new_tokens=n_tokens) + + Audio(audio_values[0].cpu().numpy(), rate=sampling_rate) + + + + +.. raw:: html + + + + + + + +Try out the converted pipeline `⇑ <#top>`__ +############################################################################################################################### + +The demo app below is created using `Gradio +package `__ + +.. code:: ipython3 + + def _generate(prompt): + inputs = processor( + text=[prompt,], + padding=True, + return_tensors="pt", + ) + audio_values = model.generate(**inputs.to(device), do_sample=True, guidance_scale=3, max_new_tokens=n_tokens) + waveform = audio_values[0].cpu().squeeze() * 2**15 + return (sampling_rate, waveform.numpy().astype(np.int16)) + +.. code:: ipython3 + + import gradio as gr + + demo = gr.Interface( + _generate, + inputs=[ + gr.Textbox(label="Text Prompt"), + ], + outputs=[ + "audio" + ], + examples=[ + ["80s pop track with bassy drums and synth"], + ["Earthy tones, environmentally conscious, ukulele-infused, harmonic, breezy, easygoing, organic instrumentation, gentle grooves"], + ["90s rock song with loud guitars and heavy drums"], + ["Heartful EDM with beautiful synths and chords"], + ], + ) + try: + demo.launch(debug=True) + except Exception: + demo.launch(share=True, debug=True) + + # If you are launching remotely, specify server_name and server_port + # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` + # To learn more please refer to the Gradio docs: https://gradio.app/docs/ diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output.rst b/docs/notebooks/251-tiny-sd-image-generation-with-output.rst new file mode 100644 index 00000000000..f8043dfe552 --- /dev/null +++ b/docs/notebooks/251-tiny-sd-image-generation-with-output.rst @@ -0,0 +1,1136 @@ +Image Generation with Tiny-SD and OpenVINO™ +=========================================== + +.. _top: + +In recent times, the AI community has witnessed a remarkable surge in +the development of larger and more performant language models, such as +Falcon 40B, LLaMa-2 70B, Falcon 40B, MPT 30B, and in the imaging domain +with models like SD2.1 and SDXL. These advancements have undoubtedly +pushed the boundaries of what AI can achieve, enabling highly versatile +and state-of-the-art image generation and language understanding +capabilities. However, the breakthrough of large models comes with +substantial computational demands. To resolve this issue, recent +research on efficient Stable Diffusion has prioritized reducing the +number of sampling steps and utilizing network quantization. + +Moving towards the goal of making image generative models faster, +smaller, and cheaper, Tiny-SD was proposed by Segmind. Tiny SD is a +compressed Stable Diffusion (SD) model that has been trained on +Knowledge-Distillation (KD) techniques and the work has been largely +based on this `paper `__. The +authors describe a Block-removal Knowledge-Distillation method where +some of the UNet layers are removed and the student model weights are +trained. Using the KD methods described in the paper, they were able to +train two compressed models using the 🧨 diffusers library; Small and +Tiny, that have 35% and 55% fewer parameters, respectively than the base +model while achieving comparable image fidelity as the base model. More +details about model can be found in `model +card `__, `blog +post `__ and training +`repository `__. + +This notebook demonstrates how to convert and run the Tiny-SD model +using OpenVINO. + +The notebook contains the following steps: + +1. Convert PyTorch models to OpenVINO Intermediate Representation using + OpenVINO Converter Tool (OVC). +2. Prepare Inference Pipeline. +3. Run Inference pipeline with OpenVINO. +4. Run Interactive demo for Tiny-SD model + +**Table of content**: + +- `Prerequisites <#prerequisites>`__ +- `Create PyTorch Models pipeline <#create-pytorch-models-pipeline>`__ +- `Convert models to OpenVINO Intermediate representation (IR) + format <#convert-models-to-openvino-intermediate-representation-format>`__ + + - `Text Encoder <#text-encoder>`__ + - `U-net <#u-net>`__ + - `VAE <#vae>`__ + +- `Prepare Inference Pipeline <#prepare-inference-pipeline>`__ +- `Configure Inference Pipeline <#configure-inference-pipeline>`__ + + - `Text-to-Image generation <#text-to-image-generation>`__ + - `Image-to-Image generation <#image-to-image-generation>`__ + +Prerequisites `⇑ <#top>`__ +############################################################################################################################### + +Install required dependencies + +.. code:: ipython3 + :force: + + ! pip install -q --find-links https://download.pytorch.org/whl/torch_stable.html \ + "torch==1.13.1; sys_platform == 'darwin'" \ + "torch==1.13.1+cpu; sys_platform == 'linux' or platform_system == 'Windows'" \ + "torchvision==0.14.1; sys_platform == 'darwin'" \ + "torchvision==0.14.1+cpu; sys_platform == 'linux' or platform_system == 'Windows'" + !pip -q install "openvino==2023.1.0.dev20230811" "diffusers>=0.18.0" "transformers>=4.30.2" "gradio" + +Create PyTorch Models pipeline `⇑ <#top>`__ +############################################################################################################################### + +``StableDiffusionPipeline`` is an end-to-end inference pipeline that you +can use to generate images from text with just a few lines of code. + +First, load the pre-trained weights of all components of the model. + +.. code:: ipython3 + + import gc + from diffusers import StableDiffusionPipeline + + model_id = "segmind/tiny-sd" + + pipe = StableDiffusionPipeline.from_pretrained(model_id).to("cpu") + text_encoder = pipe.text_encoder + text_encoder.eval() + unet = pipe.unet + unet.eval() + vae = pipe.vae + vae.eval() + + del pipe + gc.collect() + + +.. parsed-literal:: + + 2023-08-17 16:52:46.141349: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2023-08-17 16:52:46.179746: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2023-08-17 16:52:46.808512: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + vae/diffusion_pytorch_model.safetensors not found + + + +.. parsed-literal:: + + Loading pipeline components...: 0%| | 0/5 [00:00`__ +############################################################################################################################### + +OpenVINO supports PyTorch through conversion to OpenVINO Intermediate +Representation (IR) format. To take the advantage of OpenVINO +optimization tools and features, the model should be converted using the +OpenVINO Converter tool (OVC). The ``openvino.convert_model`` function +provides Python API for OVC usage. The function returns the instance of +the OpenVINO Model class, which is ready for use in the Python +interface. However, it can also be saved on disk using +``openvino.save_model`` for future execution. + +Starting from OpenVINO 2023.0.0 release OpenVINO supports direct +conversion PyTorch models. To perform conversion, we should provide +PyTorch model instance and example input into +``openvino.convert_model``. By default, model converted with dynamic +shapes preserving, in order to fixate input shape to generate image of +specific resolution, ``input`` parameter additionally can be specified. + +The model consists of three important parts: + +- Text Encoder for creation condition to generate image from text + prompt. +- U-net for step by step denoising latent image representation. +- Autoencoder (VAE) for encoding input image to latent space (if + required) and decoding latent space to image back after generation. + +Let us convert each part. + +Text Encoder `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +The text-encoder is responsible for transforming the input prompt, for +example, “a photo of an astronaut riding a horse” into an embedding +space that can be understood by the U-Net. It is usually a simple +transformer-based encoder that maps a sequence of input tokens to a +sequence of latent text embeddings. + +Input of the text encoder is the tensor ``input_ids`` which contains +indexes of tokens from text processed by tokenizer and padded to maximum +length accepted by model. Model outputs are two tensors: +``last_hidden_state`` - hidden state from the last MultiHeadAttention +layer in the model and ``pooler_out`` - Pooled output for whole model +hidden states. + +.. code:: ipython3 + + from pathlib import Path + import torch + from openvino import convert_model, save_model + + TEXT_ENCODER_OV_PATH = Path("text_encoder.xml") + + + def convert_encoder(text_encoder: torch.nn.Module, ir_path:Path): + """ + Convert Text Encoder mode. + Function accepts text encoder model, and prepares example inputs for conversion, + Parameters: + text_encoder (torch.nn.Module): text_encoder model from Stable Diffusion pipeline + ir_path (Path): File for storing model + Returns: + None + """ + input_ids = torch.ones((1, 77), dtype=torch.long) + # switch model to inference mode + text_encoder.eval() + + # disable gradients calculation for reducing memory consumption + with torch.no_grad(): + # Export model to IR format + ov_model = convert_model(text_encoder, example_input=input_ids, input=[(1,77),]) + save_model(ov_model, ir_path) + del ov_model + print(f'Text Encoder successfully converted to IR and saved to {ir_path}') + + + if not TEXT_ENCODER_OV_PATH.exists(): + convert_encoder(text_encoder, TEXT_ENCODER_OV_PATH) + else: + print(f"Text encoder will be loaded from {TEXT_ENCODER_OV_PATH}") + + del text_encoder + gc.collect() + + +.. parsed-literal:: + + Text encoder will be loaded from text_encoder.xml + + + + +.. parsed-literal:: + + 0 + + + +U-net `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +U-net model has three inputs: + +- ``sample`` - latent image sample from previous step. Generation + process has not been started yet, so you will use random noise. +- ``timestep`` - current scheduler step. +- ``encoder_hidden_state`` - hidden state of text encoder. + +Model predicts the ``sample`` state for the next step. + +.. code:: ipython3 + + import numpy as np + from openvino import PartialShape, Type + + UNET_OV_PATH = Path('unet.xml') + + dtype_mapping = { + torch.float32: Type.f32, + torch.float64: Type.f64 + } + + + def convert_unet(unet:torch.nn.Module, ir_path:Path): + """ + Convert U-net model to IR format. + Function accepts unet model, prepares example inputs for conversion, + Parameters: + unet (StableDiffusionPipeline): unet from Stable Diffusion pipeline + ir_path (Path): File for storing model + Returns: + None + """ + # prepare inputs + encoder_hidden_state = torch.ones((2, 77, 768)) + latents_shape = (2, 4, 512 // 8, 512 // 8) + latents = torch.randn(latents_shape) + t = torch.from_numpy(np.array(1, dtype=float)) + dummy_inputs = (latents, t, encoder_hidden_state) + input_info = [] + for input_tensor in dummy_inputs: + shape = PartialShape(tuple(input_tensor.shape)) + element_type = dtype_mapping[input_tensor.dtype] + input_info.append((shape, element_type)) + + unet.eval() + with torch.no_grad(): + ov_model = convert_model(unet, example_input=dummy_inputs, input=input_info) + save_model(ov_model, ir_path) + del ov_model + print(f'Unet successfully converted to IR and saved to {ir_path}') + + + if not UNET_OV_PATH.exists(): + convert_unet(unet, UNET_OV_PATH) + gc.collect() + else: + print(f"Unet will be loaded from {UNET_OV_PATH}") + del unet + gc.collect() + + +.. parsed-literal:: + + Unet will be loaded from unet.xml + + + + +.. parsed-literal:: + + 0 + + + +VAE `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +The VAE model has two parts, an encoder and a decoder. The encoder is +used to convert the image into a low dimensional latent representation, +which will serve as the input to the U-Net model. The decoder, +conversely, transforms the latent representation back into an image. + +During latent diffusion training, the encoder is used to get the latent +representations (latents) of the images for the forward diffusion +process, which applies more and more noise at each step. During +inference, the denoised latents generated by the reverse diffusion +process are converted back into images using the VAE decoder. When you +run inference for text-to-image, there is no initial image as a starting +point. You can skip this step and directly generate initial random +noise. + +As the encoder and the decoder are used independently in different parts +of the pipeline, it will be better to convert them to separate models. + +.. code:: ipython3 + + VAE_ENCODER_OV_PATH = Path("vae_encodr.xml") + + def convert_vae_encoder(vae: torch.nn.Module, ir_path: Path): + """ + Convert VAE model for encoding to IR format. + Function accepts vae model, creates wrapper class for export only necessary for inference part, + prepares example inputs for conversion, + Parameters: + vae (torch.nn.Module): VAE model from StableDiffusio pipeline + ir_path (Path): File for storing model + Returns: + None + """ + class VAEEncoderWrapper(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, image): + return self.vae.encode(x=image)["latent_dist"].sample() + vae_encoder = VAEEncoderWrapper(vae) + vae_encoder.eval() + image = torch.zeros((1, 3, 512, 512)) + with torch.no_grad(): + ov_model = convert_model(vae_encoder, example_input=image, input=[((1,3,512,512),)]) + save_model(ov_model, ir_path) + del ov_model + print(f'VAE encoder successfully converted to IR and saved to {ir_path}') + + + if not VAE_ENCODER_OV_PATH.exists(): + convert_vae_encoder(vae, VAE_ENCODER_OV_PATH) + else: + print(f"VAE encoder will be loaded from {VAE_ENCODER_OV_PATH}") + + VAE_DECODER_OV_PATH = Path('vae_decoder.xml') + + def convert_vae_decoder(vae: torch.nn.Module, ir_path: Path): + """ + Convert VAE model for decoding to IR format. + Function accepts vae model, creates wrapper class for export only necessary for inference part, + prepares example inputs for conversion, + Parameters: + vae (torch.nn.Module): VAE model frm StableDiffusion pipeline + ir_path (Path): File for storing model + Returns: + None + """ + class VAEDecoderWrapper(torch.nn.Module): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, latents): + return self.vae.decode(latents) + + vae_decoder = VAEDecoderWrapper(vae) + latents = torch.zeros((1, 4, 64, 64)) + + vae_decoder.eval() + with torch.no_grad(): + ov_model = convert_model(vae_decoder, example_input=latents, input=[((1,4,64,64),)]) + save_model(ov_model, ir_path) + del ov_model + print(f'VAE decoder successfully converted to IR and saved to {ir_path}') + + + if not VAE_DECODER_OV_PATH.exists(): + convert_vae_decoder(vae, VAE_DECODER_OV_PATH) + else: + print(f"VAE decoder will be loaded from {VAE_DECODER_OV_PATH}") + + del vae + gc.collect() + + +.. parsed-literal:: + + VAE encoder will be loaded from vae_encodr.xml + VAE decoder will be loaded from vae_decoder.xml + + + + +.. parsed-literal:: + + 0 + + + +Prepare Inference Pipeline `⇑ <#top>`__ +############################################################################################################################### + +Putting it all together, let us now take a closer look at how the model +works in inference by illustrating the logical flow. + +.. figure:: https://user-images.githubusercontent.com/29454499/260981188-c112dd0a-5752-4515-adca-8b09bea5d14a.png + :alt: sd-pipeline + + sd-pipeline + +As you can see from the diagram, the only difference between +Text-to-Image and text-guided Image-to-Image generation in approach is +how initial latent state is generated. In case of Image-to-Image +generation, you additionally have an image encoded by VAE encoder mixed +with the noise produced by using latent seed, while in Text-to-Image you +use only noise as initial latent state. The stable diffusion model takes +both a latent image representation of size :math:`64 \times 64` and a +text prompt is transformed to text embeddings of size +:math:`77 \times 768` via CLIP’s text encoder as an input. + +Next, the U-Net iteratively *denoises* the random latent image +representations while being conditioned on the text embeddings. The +output of the U-Net, being the noise residual, is used to compute a +denoised latent image representation via a scheduler algorithm. Many +different scheduler algorithms can be used for this computation, each +having its pros and cons. For Stable Diffusion, it is recommended to use +one of: + +- `PNDM + scheduler `__ +- `DDIM + scheduler `__ +- `K-LMS + scheduler `__\ (you + will use it in your pipeline) + +Theory on how the scheduler algorithm function works is out of scope for +this notebook. Nonetheless, in short, you should remember that you +compute the predicted denoised image representation from the previous +noise representation and the predicted noise residual. For more +information, refer to the recommended `Elucidating the Design Space of +Diffusion-Based Generative Models `__ + +The *denoising* process is repeated given number of times (by default +50) to step-by-step retrieve better latent image representations. When +complete, the latent image representation is decoded by the decoder part +of the variational auto encoder. + +.. code:: ipython3 + + import inspect + from typing import List, Optional, Union, Dict + + import PIL + import cv2 + + from transformers import CLIPTokenizer + from diffusers.pipelines.pipeline_utils import DiffusionPipeline + from diffusers.schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler + from openvino.runtime import Model + + + def scale_fit_to_window(dst_width:int, dst_height:int, image_width:int, image_height:int): + """ + Preprocessing helper function for calculating image size for resize with peserving original aspect ratio + and fitting image to specific window size + + Parameters: + dst_width (int): destination window width + dst_height (int): destination window height + image_width (int): source image width + image_height (int): source image height + Returns: + result_width (int): calculated width for resize + result_height (int): calculated height for resize + """ + im_scale = min(dst_height / image_height, dst_width / image_width) + return int(im_scale * image_width), int(im_scale * image_height) + + + def preprocess(image: PIL.Image.Image): + """ + Image preprocessing function. Takes image in PIL.Image format, resizes it to keep aspect ration and fits to model input window 512x512, + then converts it to np.ndarray and adds padding with zeros on right or bottom side of image (depends from aspect ratio), after that + converts data to float32 data type and change range of values from [0, 255] to [-1, 1], finally, converts data layout from planar NHWC to NCHW. + The function returns preprocessed input tensor and padding size, which can be used in postprocessing. + + Parameters: + image (PIL.Image.Image): input image + Returns: + image (np.ndarray): preprocessed image tensor + meta (Dict): dictionary with preprocessing metadata info + """ + src_width, src_height = image.size + dst_width, dst_height = scale_fit_to_window( + 512, 512, src_width, src_height) + image = np.array(image.resize((dst_width, dst_height), + resample=PIL.Image.Resampling.LANCZOS))[None, :] + pad_width = 512 - dst_width + pad_height = 512 - dst_height + pad = ((0, 0), (0, pad_height), (0, pad_width), (0, 0)) + image = np.pad(image, pad, mode="constant") + image = image.astype(np.float32) / 255.0 + image = 2.0 * image - 1.0 + image = image.transpose(0, 3, 1, 2) + return image, {"padding": pad, "src_width": src_width, "src_height": src_height} + + + class OVStableDiffusionPipeline(DiffusionPipeline): + def __init__( + self, + vae_decoder: Model, + text_encoder: Model, + tokenizer: CLIPTokenizer, + unet: Model, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + vae_encoder: Model = None, + ): + """ + Pipeline for text-to-image generation using Stable Diffusion. + Parameters: + vae (Model): + Variational Auto-Encoder (VAE) Model to decode images to and from latent representations. + text_encoder (Model): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the clip-vit-large-patch14(https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (CLIPTokenizer): + Tokenizer of class CLIPTokenizer(https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet (Model): Conditional U-Net architecture to denoise the encoded image latents. + scheduler (SchedulerMixin): + A scheduler to be used in combination with unet to denoise the encoded image latents. Can be one of + DDIMScheduler, LMSDiscreteScheduler, or PNDMScheduler. + """ + super().__init__() + self.scheduler = scheduler + self.vae_decoder = vae_decoder + self.vae_encoder = vae_encoder + self.text_encoder = text_encoder + self.unet = unet + self._text_encoder_output = text_encoder.output(0) + self._unet_output = unet.output(0) + self._vae_d_output = vae_decoder.output(0) + self._vae_e_output = vae_encoder.output(0) if vae_encoder is not None else None + self.height = 512 + self.width = 512 + self.tokenizer = tokenizer + + def __call__( + self, + prompt: Union[str, List[str]], + image: PIL.Image.Image = None, + num_inference_steps: Optional[int] = 50, + negative_prompt: Union[str, List[str]] = None, + guidance_scale: Optional[float] = 7.5, + eta: Optional[float] = 0.0, + output_type: Optional[str] = "pil", + seed: Optional[int] = None, + strength: float = 1.0, + gif: Optional[bool] = False, + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + Parameters: + prompt (str or List[str]): + The prompt or prompts to guide the image generation. + image (PIL.Image.Image, *optional*, None): + Intinal image for generation. + num_inference_steps (int, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + negative_prompt (str or List[str]): + The negative prompt or prompts to guide the image generation. + guidance_scale (float, *optional*, defaults to 7.5): + Guidance scale as defined in Classifier-Free Diffusion Guidance(https://arxiv.org/abs/2207.12598). + guidance_scale is defined as `w` of equation 2. + Higher guidance scale encourages to generate images that are closely linked to the text prompt, + usually at the expense of lower image quality. + eta (float, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [DDIMScheduler], will be ignored for others. + output_type (`str`, *optional*, defaults to "pil"): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): PIL.Image.Image or np.array. + seed (int, *optional*, None): + Seed for random generator state initialization. + gif (bool, *optional*, False): + Flag for storing all steps results or not. + Returns: + Dictionary with keys: + sample - the last generated image PIL.Image.Image or np.array + iterations - *optional* (if gif=True) images for all diffusion steps, List of PIL.Image.Image or np.array. + """ + if seed is not None: + np.random.seed(seed) + + img_buffer = [] + do_classifier_free_guidance = guidance_scale > 1.0 + # get prompt text embeddings + text_embeddings = self._encode_prompt(prompt, do_classifier_free_guidance=do_classifier_free_guidance, negative_prompt=negative_prompt) + + # set timesteps + accepts_offset = "offset" in set(inspect.signature(self.scheduler.set_timesteps).parameters.keys()) + extra_set_kwargs = {} + if accepts_offset: + extra_set_kwargs["offset"] = 1 + + self.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + latent_timestep = timesteps[:1] + + # get the initial random noise unless the user supplied it + latents, meta = self.prepare_latents(image, latent_timestep) + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if you are doing classifier free guidance + latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet([latent_model_input, t, text_embeddings])[self._unet_output] + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[0], noise_pred[1] + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(torch.from_numpy(noise_pred), t, torch.from_numpy(latents), **extra_step_kwargs)["prev_sample"].numpy() + if gif: + image = self.vae_decoder(latents * (1 / 0.18215))[self._vae_d_output] + image = self.postprocess_image(image, meta, output_type) + img_buffer.extend(image) + + # scale and decode the image latents with vae + image = self.vae_decoder(latents * (1 / 0.18215))[self._vae_d_output] + + image = self.postprocess_image(image, meta, output_type) + return {"sample": image, 'iterations': img_buffer} + + def _encode_prompt(self, prompt:Union[str, List[str]], num_images_per_prompt:int = 1, do_classifier_free_guidance:bool = True, negative_prompt:Union[str, List[str]] = None): + """ + Encodes the prompt into text encoder hidden states. + + Parameters: + prompt (str or list(str)): prompt to be encoded + num_images_per_prompt (int): number of images that should be generated per prompt + do_classifier_free_guidance (bool): whether to use classifier free guidance or not + negative_prompt (str or list(str)): negative prompt to be encoded + Returns: + text_embeddings (np.ndarray): text encoder hidden states + """ + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + # tokenize input prompts + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + + text_embeddings = self.text_encoder( + text_input_ids)[self._text_encoder_output] + + # duplicate text embeddings for each generation per prompt + if num_images_per_prompt != 1: + bs_embed, seq_len, _ = text_embeddings.shape + text_embeddings = np.tile( + text_embeddings, (1, num_images_per_prompt, 1)) + text_embeddings = np.reshape( + text_embeddings, (bs_embed * num_images_per_prompt, seq_len, -1)) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_tokens: List[str] + max_length = text_input_ids.shape[-1] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + else: + uncond_tokens = negative_prompt + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + + uncond_embeddings = self.text_encoder(uncond_input.input_ids)[self._text_encoder_output] + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = uncond_embeddings.shape[1] + uncond_embeddings = np.tile(uncond_embeddings, (1, num_images_per_prompt, 1)) + uncond_embeddings = np.reshape(uncond_embeddings, (batch_size * num_images_per_prompt, seq_len, -1)) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_embeddings = np.concatenate([uncond_embeddings, text_embeddings]) + + return text_embeddings + + + def prepare_latents(self, image:PIL.Image.Image = None, latent_timestep:torch.Tensor = None): + """ + Function for getting initial latents for starting generation + + Parameters: + image (PIL.Image.Image, *optional*, None): + Input image for generation, if not provided randon noise will be used as starting point + latent_timestep (torch.Tensor, *optional*, None): + Predicted by scheduler initial step for image generation, required for latent image mixing with nosie + Returns: + latents (np.ndarray): + Image encoded in latent space + """ + latents_shape = (1, 4, self.height // 8, self.width // 8) + noise = np.random.randn(*latents_shape).astype(np.float32) + if image is None: + # if you use LMSDiscreteScheduler, let's make sure latents are multiplied by sigmas + if isinstance(self.scheduler, LMSDiscreteScheduler): + noise = noise * self.scheduler.sigmas[0].numpy() + return noise, {} + input_image, meta = preprocess(image) + latents = self.vae_encoder(input_image)[self._vae_e_output] * 0.18215 + latents = self.scheduler.add_noise(torch.from_numpy(latents), torch.from_numpy(noise), latent_timestep).numpy() + return latents, meta + + def postprocess_image(self, image:np.ndarray, meta:Dict, output_type:str = "pil"): + """ + Postprocessing for decoded image. Takes generated image decoded by VAE decoder, unpad it to initila image size (if required), + normalize and convert to [0, 255] pixels range. Optionally, convertes it from np.ndarray to PIL.Image format + + Parameters: + image (np.ndarray): + Generated image + meta (Dict): + Metadata obtained on latents preparing step, can be empty + output_type (str, *optional*, pil): + Output format for result, can be pil or numpy + Returns: + image (List of np.ndarray or PIL.Image.Image): + Postprocessed images + """ + if "padding" in meta: + pad = meta["padding"] + (_, end_h), (_, end_w) = pad[1:3] + h, w = image.shape[2:] + unpad_h = h - end_h + unpad_w = w - end_w + image = image[:, :, :unpad_h, :unpad_w] + image = np.clip(image / 2 + 0.5, 0, 1) + image = np.transpose(image, (0, 2, 3, 1)) + # 9. Convert to PIL + if output_type == "pil": + image = self.numpy_to_pil(image) + if "src_height" in meta: + orig_height, orig_width = meta["src_height"], meta["src_width"] + image = [img.resize((orig_width, orig_height), + PIL.Image.Resampling.LANCZOS) for img in image] + else: + if "src_height" in meta: + orig_height, orig_width = meta["src_height"], meta["src_width"] + image = [cv2.resize(img, (orig_width, orig_width)) + for img in image] + return image + + def get_timesteps(self, num_inference_steps:int, strength:float): + """ + Helper function for getting scheduler timesteps for generation + In case of image-to-image generation, it updates number of steps according to strength + + Parameters: + num_inference_steps (int): + number of inference steps for generation + strength (float): + value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. + Values that approach 1.0 enable lots of variations but will also produce images that are not semantically consistent with the input. + """ + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + +Configure Inference Pipeline +---------------------------- + +First, you should create instances of OpenVINO Model. + +.. code:: ipython3 + + from openvino.runtime import Core + core = Core() + +Select device from dropdown list for running inference using OpenVINO. + +.. code:: ipython3 + + import ipywidgets as widgets + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value='AUTO', + description='Device:', + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=2, options=('CPU', 'GPU', 'AUTO'), value='AUTO') + + + +.. code:: ipython3 + + text_enc = core.compile_model(TEXT_ENCODER_OV_PATH, device.value) + +.. code:: ipython3 + + unet_model = core.compile_model(UNET_OV_PATH, device.value) + +.. code:: ipython3 + + vae_decoder = core.compile_model(VAE_DECODER_OV_PATH, device.value) + vae_encoder = core.compile_model(VAE_ENCODER_OV_PATH, device.value) + +Model tokenizer and scheduler are also important parts of the pipeline. +Let us define them and put all components together + +.. code:: ipython3 + + from transformers import CLIPTokenizer + from diffusers.schedulers import LMSDiscreteScheduler + + lms = LMSDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear" + ) + tokenizer = CLIPTokenizer.from_pretrained('openai/clip-vit-large-patch14') + + ov_pipe = OVStableDiffusionPipeline( + tokenizer=tokenizer, + text_encoder=text_enc, + unet=unet_model, + vae_encoder=vae_encoder, + vae_decoder=vae_decoder, + scheduler=lms + ) + +Text-to-Image generation `⇑ <#top>`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Now, let’s see model in action + +.. code:: ipython3 + + text_prompt = 'RAW studio photo of An intricate forest minitown landscape trapped in a bottle, atmospheric oliva lighting, on the table, intricate details, dark shot, soothing tones, muted colors ' + seed = 431 + num_steps = 20 + +.. code:: ipython3 + + print('Pipeline settings') + print(f'Input text: {text_prompt}') + print(f'Seed: {seed}') + print(f'Number of steps: {num_steps}') + + +.. parsed-literal:: + + Pipeline settings + Input text: RAW studio photo of An intricate forest minitown landscape trapped in a bottle, atmospheric oliva lighting, on the table, intricate details, dark shot, soothing tones, muted colors + Seed: 431 + Number of steps: 20 + + +.. code:: ipython3 + + result = ov_pipe(text_prompt, num_inference_steps=num_steps, seed=seed) + + + +.. parsed-literal:: + + 0%| | 0/20 [00:00`__ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +One of the most amazing features of Stable Diffusion model is the +ability to condition image generation from an existing image or sketch. +Given a (potentially crude) image and the right text prompt, latent +diffusion models can be used to “enhance” an image. + +Image-to-Image generation, in additionally to the text prompt, requires +providing the initial image. Optionally, you can also change +``strength`` parameter, which is a value between 0.0 and 1.0, that +controls the amount of noise that is added to the input image. Values +that approach 1.0 enable lots of variations but will also produce images +that are not semantically consistent with the input. One of the +interesting use cases for Image-to-Image generation is depainting - +turning sketches or paintings into realistic photographs. + +Additionally, to improve image generation quality, model supports +negative prompting. Technically, positive prompt steers the diffusion +toward the images associated with it, while negative prompt steers the +diffusion away from it.In other words, negative prompt declares +undesired concepts for generation image, e.g. if we want to have +colorful and bright image, gray scale image will be result which we want +to avoid, in this case gray scale can be treated as negative prompt. The +positive and negative prompt are in equal footing. You can always use +one with or without the other. More explanation of how it works can be +found in this +`article `__. + +.. code:: ipython3 + + text_prompt_i2i = 'professional photo portrait of woman, highly detailed, hyper realistic, cinematic effects, soft lighting' + negative_prompt_i2i = "blurry, poor quality, low res, worst quality, cropped, ugly, poorly drawn face, without eyes, mutation, unreal, animate, poorly drawn eyes" + num_steps_i2i = 40 + seed_i2i = 82698152 + strength = 0.68 + +.. code:: ipython3 + + from diffusers.utils import load_image + + default_image_url = "https://user-images.githubusercontent.com/29454499/260418860-69cc443a-9ee6-493c-a393-3a97af080be7.jpg" + # read uploaded image + image = load_image(default_image_url) + print('Pipeline settings') + print(f'Input positive prompt: \n\t{text_prompt_i2i}') + print(f'Input negative prompt: \n\t{negative_prompt_i2i}') + print(f'Seed: {seed_i2i}') + print(f'Number of steps: {num_steps_i2i}') + print(f'Strength: {strength}') + print("Input image:") + display(image) + processed_image = ov_pipe(text_prompt_i2i, image, negative_prompt=negative_prompt_i2i, num_inference_steps=num_steps_i2i, seed=seed_i2i, strength=strength) + + +.. parsed-literal:: + + Pipeline settings + Input positive prompt: + professional photo portrait of woman, highly detailed, hyper realistic, cinematic effects, soft lighting + Input negative prompt: + blurry, poor quality, low res, worst quality, cropped, ugly, poorly drawn face, without eyes, mutation, unreal, animate, poorly drawn eyes + Seed: 82698152 + Number of steps: 40 + Strength: 0.68 + Input image: + + + +.. image:: 251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_37_1.png + + + +.. parsed-literal:: + + 0%| | 0/27 [00:00`__ +############################################################################################################################### + +.. code:: ipython3 + + import gradio as gr + + sample_img_url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/tower.jpg" + + img = load_image(sample_img_url).save("tower.jpg") + + def generate_from_text(text, negative_text, seed, num_steps, _=gr.Progress(track_tqdm=True)): + result = ov_pipe(text, negative_prompt=negative_text, num_inference_steps=num_steps, seed=seed) + return result["sample"][0] + + + def generate_from_image(img, text, negative_text, seed, num_steps, strength, _=gr.Progress(track_tqdm=True)): + result = ov_pipe(text, img, negative_prompt=negative_text, num_inference_steps=num_steps, seed=seed, strength=strength) + return result["sample"][0] + + + with gr.Blocks() as demo: + with gr.Tab("Text-to-Image generation"): + with gr.Row(): + with gr.Column(): + text_input = gr.Textbox(lines=3, label="Positive prompt") + negative_text_input = gr.Textbox(lines=3, label="Negative prompt") + seed_input = gr.Slider(0, 10000000, value=751, label="Seed") + steps_input = gr.Slider(1, 50, value=20, step=1, label="Steps") + out = gr.Image(label="Result", type="pil") + sample_text = "futuristic synthwave city, retro sunset, crystals, spires, volumetric lighting, studio Ghibli style, rendered in unreal engine with clean details" + sample_text2 = "RAW studio photo of tiny cute happy cat in a yellow raincoat in the woods, rain, a character portrait, soft lighting, high resolution, photo realistic, extremely detailed" + negative_sample_text = "" + negative_sample_text2 = "bad anatomy, blurry, noisy, jpeg artifacts, low quality, geometry, mutation, disgusting. ugly" + btn = gr.Button() + btn.click(generate_from_text, [text_input, negative_text_input, seed_input, steps_input], out) + gr.Examples([[sample_text, negative_sample_text, 42, 20], [sample_text2, negative_sample_text2, 1561, 25]], [text_input, negative_text_input, seed_input, steps_input]) + with gr.Tab("Image-to-Image generation"): + with gr.Row(): + with gr.Column(): + i2i_input = gr.Image(label="Image", type="pil") + i2i_text_input = gr.Textbox(lines=3, label="Text") + i2i_negative_text_input = gr.Textbox(lines=3, label="Negative prompt") + i2i_seed_input = gr.Slider(0, 10000000, value=42, label="Seed") + i2i_steps_input = gr.Slider(1, 50, value=10, step=1, label="Steps") + strength_input = gr.Slider(0, 1, value=0.5, label="Strength") + i2i_out = gr.Image(label="Result", type="pil") + i2i_btn = gr.Button() + sample_i2i_text = "amazing watercolor painting" + i2i_btn.click( + generate_from_image, + [i2i_input, i2i_text_input, i2i_negative_text_input, i2i_seed_input, i2i_steps_input, strength_input], + i2i_out, + ) + gr.Examples( + [["tower.jpg", sample_i2i_text, "", 6400023, 40, 0.3]], + [i2i_input, i2i_text_input, i2i_negative_text_input, i2i_seed_input, i2i_steps_input, strength_input], + ) + + try: + demo.queue().launch(debug=True) + except Exception: + demo.queue().launch(share=True, debug=True) + # if you are launching remotely, specify server_name and server_port + # demo.launch(server_name='your server name', server_port='server port in int') + # Read more in the docs: https://gradio.app/docs/ + + +.. parsed-literal:: + + Running on local URL: http://127.0.0.1:7860 + + To create a public link, set `share=True` in `launch()`. + + + +.. raw:: html + +
+ diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_33_1.png b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_33_1.png new file mode 100644 index 00000000000..2826eba480a --- /dev/null +++ b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_33_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d4313551a427c44d4bba572717be2134c99e0e8785a8f11d5fd2993ccfb10da +size 470354 diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_37_1.png b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_37_1.png new file mode 100644 index 00000000000..7696d04c750 --- /dev/null +++ b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_37_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97e1e206d64979d8fc9aaa6e46786172062c9a9eb1817a351af2a38557a012f +size 835162 diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_39_1.png b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_39_1.png new file mode 100644 index 00000000000..d4d0bdaebad --- /dev/null +++ b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/251-tiny-sd-image-generation_39_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e4fb5a7c284be4de90e648badd409935c29b561b30f728ff905f522dd5a99f9 +size 699062 diff --git a/docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html b/docs/notebooks/251-tiny-sd-image-generation-with-output_files/index.html new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/notebooks/notebooks_with_binder_buttons.txt b/docs/notebooks/notebooks_with_binder_buttons.txt index 6a70fccc790..1ea3afbe1bc 100644 --- a/docs/notebooks/notebooks_with_binder_buttons.txt +++ b/docs/notebooks/notebooks_with_binder_buttons.txt @@ -30,6 +30,7 @@ 229-distilbert-sequence-classification 243-tflite-selfie-segmentation 247-code-language-id +250-music-generation 401-object-detection 402-pose-estimation 403-action-recognition-webcam diff --git a/docs/notebooks/notebooks_with_colab_buttons.txt b/docs/notebooks/notebooks_with_colab_buttons.txt index 695560c6ad0..151267b3675 100644 --- a/docs/notebooks/notebooks_with_colab_buttons.txt +++ b/docs/notebooks/notebooks_with_colab_buttons.txt @@ -21,6 +21,8 @@ 232-clip-language-saliency-map 243-tflite-selfie-segmentation 244-named-entity-recognition +250-music-generation +251-tiny-sd-image-generation 305-tensorflow-quantization-aware-training 401-object-detection 404-style-transfer diff --git a/docs/tutorials.md b/docs/tutorials.md index f59b187c054..2b0a08e47d1 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -257,6 +257,10 @@ Demos that demonstrate inference on a particular model. +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `249-oneformer-segmentation `__ | Universal segmentation with OneFormer and OpenVINO™. | |n249-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ + | `250-music-generation `__ |br| |n250| |br| |c250| | Controllable Music Generation with MusicGen and OpenVINO™. | |n250-img1| | + +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ + | `251-tiny-sd-image-generation `__ |br| |c251| | Image Generation with Tiny-SD and OpenVINO™. | |n251-img1| | + +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ Model Training @@ -318,7 +322,7 @@ The following tutorials are guaranteed to provide a great experience with infere +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `Whisper - Subtitles generation `__ |br| |c227| | Generate subtitles for video with OpenAI Whisper and OpenVINO. | |n227-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `CLIP - zero-shot-image-classification `__ | Perform Zero-shot image classification with CLIP and OpenVINO. | |n228-img1| | +| `CLIP - zero-shot-image-classification `__ | Perform Zero-shot image classification with CLIP and OpenVINO. | |n228-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `BLIP - Visual-language-processing `__ | Visual question answering and image captioning using BLIP and OpenVINO™. | |n233-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -326,11 +330,15 @@ The following tutorials are guaranteed to provide a great experience with infere +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `DeepFloyd IF - Text-to-Image generation `__ | Text-to-image generation with DeepFloyd IF and OpenVINO™. | |n238-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `ImageBind `__ | Binding multimodal data, using ImageBind and OpenVINO™. | |n239-img1| | +| `ImageBind `__ | Binding multimodal data, using ImageBind and OpenVINO™. | |n239-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `Dolly v2 `__ | Instruction following using Databricks Dolly 2.0 and OpenVINO™. | |n240-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `248-stable-diffusion-xl `__ | Image generation with Stable Diffusion XL and OpenVINO™. | |n248-img1| | +| `Stable Diffusion XL `__ | Image generation with Stable Diffusion XL and OpenVINO™. | |n248-img1| | ++-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ +| `MusicGen `__ |br| |n250| |br| |c250| | Controllable Music Generation with MusicGen and OpenVINO™. | |n250-img1| | ++-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ +| `Tiny SD `__ |br| |c251| | Image Generation with Tiny-SD and OpenVINO™. | |n251-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -487,6 +495,10 @@ Made with `contributors-img `__. :target: https://user-images.githubusercontent.com/29454499/258651862-28b63016-c5ff-4263-9da8-73ca31100165.jpeg .. |n249-img1| image:: https://camo.githubusercontent.com/f46c3642d3266e9d56d8ea8a943e67825597de3ff51698703ea2ddcb1086e541/68747470733a2f2f6769746875622d70726f64756374696f6e2d757365722d61737365742d3632313064662e73332e616d617a6f6e6177732e636f6d2f37363136313235362f3235383634303731332d66383031626430392d653932372d346162642d616132662d3939393064653463616638642e676966 :target: https://camo.githubusercontent.com/f46c3642d3266e9d56d8ea8a943e67825597de3ff51698703ea2ddcb1086e541/68747470733a2f2f6769746875622d70726f64756374696f6e2d757365722d61737365742d3632313064662e73332e616d617a6f6e6177732e636f6d2f37363136313235362f3235383634303731332d66383031626430392d653932372d346162642d616132662d3939393064653463616638642e676966 +.. |n250-img1| image:: https://user-images.githubusercontent.com/76463150/260439306-81c81c8d-1f9c-41d0-b881-9491766def8e.png + :target: https://user-images.githubusercontent.com/76463150/260439306-81c81c8d-1f9c-41d0-b881-9491766def8e.png +.. |n251-img1| image:: https://user-images.githubusercontent.com/29454499/260904650-274fc2f9-24d2-46a3-ac3d-d660ec3c9a19.png + :target: https://user-images.githubusercontent.com/29454499/260904650-274fc2f9-24d2-46a3-ac3d-d660ec3c9a19.png .. |n301-img1| image:: https://user-images.githubusercontent.com/15709723/127779607-8fa34947-1c35-4260-8d04-981c41a2a2cc.png :target: https://user-images.githubusercontent.com/15709723/127779607-8fa34947-1c35-4260-8d04-981c41a2a2cc.png .. |n401-img1| image:: https://user-images.githubusercontent.com/4547501/141471665-82b28c86-cf64-4bfe-98b3-c314658f2d96.gif @@ -647,6 +659,12 @@ Made with `contributors-img `__. :target: https://colab.research.google.com/github/openvinotoolkit/openvino_notebooks/blob/main/notebooks/244-named-entity-recognition/244-named-entity-recognition.ipynb .. |n247| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F247-code-language-id%2F247-code-language-id.ipynb +.. |n250| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F250-music-generation%2F250-music-generation.ipynb +.. |c250| image:: https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667 + :target: https://colab.research.google.com/github/openvinotoolkit/openvino_notebooks/blob/main/notebooks/250-music-generation/250-music-generation.ipynb +.. |c251| image:: https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667 + :target: https://colab.research.google.com/github/openvinotoolkit/openvino_notebooks/blob/main/notebooks/251-tiny-sd-image-generation/251-tiny-sd-image-generation.ipynb .. |c305| image:: https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667 :width: 109 :target: https://colab.research.google.com/github/openvinotoolkit/openvino_notebooks/blob/main/notebooks/305-tensorflow-quantization-aware-training/305-tensorflow-quantization-aware-training.ipynb