diff --git a/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp
index c3aae62596a..f6182f14733 100644
--- a/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp
+++ b/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp
@@ -113,7 +113,7 @@ ngraph::pass::FakeQuantizeMulFusion::FakeQuantizeMulFusion() {
                                                                  fq_node->input_value(2),
                                                                  get_adjusted_output_range(original_output_low),
                                                                  get_adjusted_output_range(original_output_high)});
-        bool fq_on_weights = is_type<opset4::Constant>(data.get_node());
+        bool fq_on_weights = is_type<opset4::Constant>(data.get_node()) || get_constant_from_source(data) != nullptr;
         if (!fq_on_weights && transformation_callback(new_fq_node))
             return false;
 
diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/quantized_convolution_batch_norm.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/quantized_convolution_batch_norm.cpp
index ad0818b4442..77ba5714cc9 100644
--- a/src/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/quantized_convolution_batch_norm.cpp
+++ b/src/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/quantized_convolution_batch_norm.cpp
@@ -9,7 +9,21 @@ using namespace SubgraphTestsDefinitions;
 namespace {
 
 INSTANTIATE_TEST_SUITE_P(smoke_QuantizedConvolutionBatchNorm, QuantizedConvolutionBatchNorm,
-                         ::testing::Values(CommonTestUtils::DEVICE_CPU),
+                         ::testing::Combine(
+                             ::testing::ValuesIn({
+                                 QuantizeType::FAKE_QUANTIZE,
+                                 QuantizeType::QUANTIZE_DEQUANTIZE,
+                                 QuantizeType::COMPRESSED_WEIGHTS}),
+                             ::testing::Values(false),
+                             ::testing::Values(CommonTestUtils::DEVICE_CPU)),
+                         QuantizedConvolutionBatchNorm::getTestCaseName);
+
+INSTANTIATE_TEST_SUITE_P(smoke_QuantizedConvolutionBatchNormTransposeOnWeights, QuantizedConvolutionBatchNorm,
+                         ::testing::Combine(
+                             ::testing::ValuesIn({
+                                 QuantizeType::FAKE_QUANTIZE}),
+                             ::testing::Values(true),
+                             ::testing::Values(CommonTestUtils::DEVICE_CPU)),
                          QuantizedConvolutionBatchNorm::getTestCaseName);
 
 } // namespace
diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/quantized_convolution_batch_norm.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/quantized_convolution_batch_norm.hpp
index c31b681b5fb..3b9eedca180 100644
--- a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/quantized_convolution_batch_norm.hpp
+++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/quantized_convolution_batch_norm.hpp
@@ -8,10 +8,19 @@
 
 namespace SubgraphTestsDefinitions {
 
-class QuantizedConvolutionBatchNorm : public testing::WithParamInterface<std::string>,
+enum class QuantizeType {
+    FAKE_QUANTIZE,
+    QUANTIZE_DEQUANTIZE,
+    COMPRESSED_WEIGHTS,
+    COMPRESSED_WEIGHTS_NO_SHIFT,
+};
+
+using QuantizedConvolutionBatchNormParams = std::tuple<QuantizeType, bool, std::string>;
+
+class QuantizedConvolutionBatchNorm : public testing::WithParamInterface<QuantizedConvolutionBatchNormParams>,
                                             virtual public LayerTestsUtils::LayerTestsCommon {
 public:
-    static std::string getTestCaseName(const testing::TestParamInfo<std::string>& obj);
+    static std::string getTestCaseName(const testing::TestParamInfo<QuantizedConvolutionBatchNormParams>& obj);
 
 protected:
     void SetUp() override;
diff --git a/src/tests/functional/shared_test_classes/src/subgraph/quantized_convolution_batch_norm.cpp b/src/tests/functional/shared_test_classes/src/subgraph/quantized_convolution_batch_norm.cpp
index b1cf3445ca7..ace00182cea 100644
--- a/src/tests/functional/shared_test_classes/src/subgraph/quantized_convolution_batch_norm.cpp
+++ b/src/tests/functional/shared_test_classes/src/subgraph/quantized_convolution_batch_norm.cpp
@@ -8,23 +8,97 @@
 
 namespace SubgraphTestsDefinitions {
 
-std::string QuantizedConvolutionBatchNorm::getTestCaseName(const testing::TestParamInfo<std::string>& obj) {
-    return "device=" + obj.param;
+std::ostream& operator<<(std::ostream& os, QuantizeType type) {
+    switch (type) {
+        case QuantizeType::FAKE_QUANTIZE:
+            os << "fake_quantize";
+            break;
+        case QuantizeType::QUANTIZE_DEQUANTIZE:
+            os << "quantize_dequantize";
+            break;
+        case QuantizeType::COMPRESSED_WEIGHTS:
+            os << "compressed_weights";
+            break;
+    }
+    return os;
+}
+
+std::string QuantizedConvolutionBatchNorm::getTestCaseName(const testing::TestParamInfo<QuantizedConvolutionBatchNormParams>& obj) {
+    std::ostringstream name;
+    QuantizeType quantize_type;
+    bool transpose_on_weights;
+    std::string device;
+    std::tie(quantize_type, transpose_on_weights, device) = obj.param;
+    name << "quantize_type=" << quantize_type << "_transpose_on_weights=" << std::boolalpha << transpose_on_weights << "_device=" << device;
+    return name.str();
 }
 
 using namespace ngraph;
 
 void QuantizedConvolutionBatchNorm::SetUp() {
-    targetDevice = GetParam();
+    QuantizeType quantize_type;
+    bool transpose_on_weights;
+    std::tie(quantize_type, transpose_on_weights, targetDevice) = GetParam();
 
     auto parameter = std::make_shared<opset8::Parameter>(element::f32, Shape{1, 16, 14, 14});
     auto low_act = opset8::Constant::create(element::f32, Shape{}, {0});
     auto high_act = opset8::Constant::create(element::f32, Shape{}, {20});
-    std::shared_ptr<Node> activations = std::make_shared<opset8::FakeQuantize>(parameter, low_act, high_act, low_act, high_act, 256);
+    std::shared_ptr<Node> activations = nullptr;
     auto low_weights = opset8::Constant::create(element::f32, Shape{}, {-0.72519057});
     auto high_weights = opset8::Constant::create(element::f32, Shape{}, {0.72519057});
-    std::shared_ptr<Node> weights = builder::makeConstant(element::f32, Shape{16, 16, 1, 1}, {}, true, 0.5f, -0.5f);
-    weights = std::make_shared<opset8::FakeQuantize>(weights, low_weights, high_weights, low_weights, high_weights, 255);
+    std::shared_ptr<Node> weights = nullptr;
+    if (transpose_on_weights)
+        weights = builder::makeConstant(element::f32, Shape{1, 16, 1, 16}, {}, true, 0.5f, -0.5f);
+    else
+        weights = builder::makeConstant(element::f32, Shape{16, 16, 1, 1}, {}, true, 0.5f, -0.5f);
+    if (quantize_type == QuantizeType::FAKE_QUANTIZE) {
+        activations = std::make_shared<opset8::FakeQuantize>(parameter, low_act, high_act, low_act, high_act, 256);
+        weights = std::make_shared<opset8::FakeQuantize>(weights, low_weights, high_weights, low_weights, high_weights, 255);
+    } else if (quantize_type == QuantizeType::QUANTIZE_DEQUANTIZE) {
+        auto output_low_act = opset8::Constant::create(element::f32, Shape{}, {0});
+        auto output_high_act = opset8::Constant::create(element::f32, Shape{}, {255});
+        activations = std::make_shared<opset8::FakeQuantize>(parameter, low_act, high_act, output_low_act, output_high_act, 256);
+        activations = std::make_shared<opset8::Convert>(activations, element::u8);
+        activations = std::make_shared<opset8::Convert>(activations, element::f32);
+        std::shared_ptr<Node> shift_act = opset8::Constant::create(element::u8, Shape{}, {0});
+        shift_act = std::make_shared<opset8::Convert>(shift_act, element::f32);
+        activations = std::make_shared<opset8::Subtract>(activations, shift_act);
+        auto scale_act = opset8::Constant::create(element::f32, Shape{}, {20.0 / 255.0});
+        activations = std::make_shared<opset8::Multiply>(activations, scale_act);
+
+        auto output_low_weights = opset8::Constant::create(element::f32, Shape{}, {0});
+        auto output_high_weights = opset8::Constant::create(element::f32, Shape{}, {254});
+        weights = std::make_shared<opset8::FakeQuantize>(weights, low_weights, high_weights, output_low_weights, output_high_weights, 255);
+        weights = std::make_shared<opset8::Convert>(weights, element::i8);
+        weights = std::make_shared<opset8::Convert>(weights, element::f32);
+        std::shared_ptr<Node> shift_weights = opset8::Constant::create(element::i8, Shape{}, {0});
+        shift_weights = std::make_shared<opset8::Convert>(shift_weights, element::f32);
+        weights = std::make_shared<opset8::Subtract>(weights, shift_weights);
+        auto scale_weights = opset8::Constant::create(element::f32, Shape{}, {2.0 / 255.0});
+        weights = std::make_shared<opset8::Multiply>(weights, scale_weights);
+    } else if (quantize_type == QuantizeType::COMPRESSED_WEIGHTS) {
+        auto output_low_act = opset8::Constant::create(element::f32, Shape{}, {0});
+        auto output_high_act = opset8::Constant::create(element::f32, Shape{}, {255});
+        activations = std::make_shared<opset8::FakeQuantize>(parameter, low_act, high_act, output_low_act, output_high_act, 256);
+        activations = std::make_shared<opset8::Convert>(activations, element::u8);
+        activations = std::make_shared<opset8::Convert>(activations, element::f32);
+        std::shared_ptr<Node> shift_act = opset8::Constant::create(element::u8, Shape{}, {0});
+        shift_act = std::make_shared<opset8::Convert>(shift_act, element::f32);
+        activations = std::make_shared<opset8::Subtract>(activations, shift_act);
+        auto scale_act = opset8::Constant::create(element::f32, Shape{}, {20.0 / 255.0});
+        activations = std::make_shared<opset8::Multiply>(activations, scale_act);
+
+        auto output_low_weights = opset8::Constant::create(element::f32, Shape{}, {0});
+        auto output_high_weights = opset8::Constant::create(element::f32, Shape{}, {254});
+        weights = std::make_shared<opset8::FakeQuantize>(weights, low_weights, high_weights, output_low_weights, output_high_weights, 255);
+        weights = std::make_shared<opset8::Convert>(weights, element::i8);
+        weights = get_constant_from_source(weights);
+        weights = std::make_shared<opset8::Convert>(weights, element::f32);
+        auto scale_weights = opset8::Constant::create(element::f32, Shape{}, {2.0 / 255.0});
+        weights = std::make_shared<opset8::Multiply>(weights, scale_weights);
+    }
+    if (transpose_on_weights)
+        weights = std::make_shared<opset8::Transpose>(weights, opset8::Constant::create(element::i32, Shape{4}, {1, 3, 0, 2}));
     auto conv = std::make_shared<opset8::Convolution>(activations, weights, Strides{1, 1}, CoordinateDiff{0, 0}, CoordinateDiff{0, 0}, Strides{1, 1});
     auto gamma = builder::makeConstant(element::f32, Shape{16}, {}, true, 1.0f, 0.1f);
     auto beta = builder::makeConstant(element::f32, Shape{16}, {}, true, 1.0f, 0.1f);