diff --git a/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp b/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp
index c33b88b09d3..b6d08888c23 100644
--- a/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp
+++ b/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp
@@ -75,7 +75,16 @@ ngraph::pass::GroupedGatherElimination::GroupedGatherElimination() {
                 (curr->input_value(0) != next->input_value(0))) {
                 ++i;
                 continue;
-            }  // curr and next are the same type of gather which takes data from the same source
+            }
+
+            // Scalar inputs are not supported by Concat and we don't want to throw an exception here.
+            // The transformation should not be applied instead.
+            if (curr->input_value(1).get_partial_shape().same_scheme(Shape{}) ||
+                next->input_value(1).get_partial_shape().same_scheme(Shape{})) {
+                return false;
+            }
+
+            // curr and next are the same type of gather which takes data from the same source
             auto joint_indices = ngraph::op::util::make_try_fold<opset1::Concat>(
                 OutputVector{curr->input_value(1), next->input_value(1)},
                 0);