diff --git a/module/Installer/src/Config/Plugin/ApplicationConfigCustomizer.php b/module/Installer/src/Config/Plugin/ApplicationConfigCustomizer.php
index c7987284..e6c02a98 100644
--- a/module/Installer/src/Config/Plugin/ApplicationConfigCustomizer.php
+++ b/module/Installer/src/Config/Plugin/ApplicationConfigCustomizer.php
@@ -4,10 +4,13 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\Installer\Config\Plugin;
 
 use Shlinkio\Shlink\Common\Util\StringUtilsTrait;
+use Shlinkio\Shlink\Installer\Exception\InvalidConfigOptionException;
 use Shlinkio\Shlink\Installer\Model\CustomizableAppConfig;
 use Symfony\Component\Console\Style\SymfonyStyle;
 use function array_diff;
 use function array_keys;
+use function is_numeric;
+use function sprintf;
 
 class ApplicationConfigCustomizer implements ConfigCustomizerInterface
 {
@@ -64,12 +67,24 @@ class ApplicationConfigCustomizer implements ConfigCustomizerInterface
                     . 'have more than a specific amount of visits?'
                 );
             case self::VISITS_THRESHOLD:
-                return (int) $io->ask(
+                return $io->ask(
                     'What is the amount of visits from which the system will not allow short URLs to be deleted?',
-                    15
+                    15,
+                    [$this, 'validateVisitsThreshold']
                 );
         }
 
         return '';
     }
+
+    public function validateVisitsThreshold($value): int
+    {
+        if (! is_numeric($value) || $value < 1) {
+            throw new InvalidConfigOptionException(
+                sprintf('Provided value "%s" is invalid. Expected a number greater than 1', $value)
+            );
+        }
+
+        return (int) $value;
+    }
 }
diff --git a/module/Installer/src/Exception/InvalidConfigOptionException.php b/module/Installer/src/Exception/InvalidConfigOptionException.php
new file mode 100644
index 00000000..647f8edc
--- /dev/null
+++ b/module/Installer/src/Exception/InvalidConfigOptionException.php
@@ -0,0 +1,10 @@
+<?php
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Installer\Exception;
+
+use RuntimeException;
+
+class InvalidConfigOptionException extends RuntimeException implements ExceptionInterface
+{
+}
diff --git a/module/Installer/test/Config/Plugin/ApplicationConfigCustomizerTest.php b/module/Installer/test/Config/Plugin/ApplicationConfigCustomizerTest.php
index 3fbcf449..d5b162f2 100644
--- a/module/Installer/test/Config/Plugin/ApplicationConfigCustomizerTest.php
+++ b/module/Installer/test/Config/Plugin/ApplicationConfigCustomizerTest.php
@@ -7,8 +7,11 @@ use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\Installer\Config\Plugin\ApplicationConfigCustomizer;
+use Shlinkio\Shlink\Installer\Exception\InvalidConfigOptionException;
 use Shlinkio\Shlink\Installer\Model\CustomizableAppConfig;
 use Symfony\Component\Console\Style\SymfonyStyle;
+use function array_shift;
+use function strpos;
 
 class ApplicationConfigCustomizerTest extends TestCase
 {
@@ -34,17 +37,47 @@ class ApplicationConfigCustomizerTest extends TestCase
      */
     public function configIsRequestedToTheUser()
     {
-        $ask = $this->io->ask(Argument::cetera())->willReturn('the_secret');
+        $ask = $this->io->ask(Argument::cetera())->willReturn('asked');
+        $confirm = $this->io->confirm(Argument::cetera())->willReturn(false);
+
         $config = new CustomizableAppConfig();
 
         $this->plugin->process($this->io->reveal(), $config);
 
         $this->assertTrue($config->hasApp());
         $this->assertEquals([
-            'SECRET' => 'the_secret',
-            'DISABLE_TRACK_PARAM' => 'the_secret',
+            'SECRET' => 'asked',
+            'DISABLE_TRACK_PARAM' => 'asked',
+            'CHECK_VISITS_THRESHOLD' => false,
         ], $config->getApp());
         $ask->shouldHaveBeenCalledTimes(2);
+        $confirm->shouldHaveBeenCalledTimes(1);
+    }
+
+    /**
+     * @test
+     */
+    public function visitsThresholdIsRequestedIfCheckIsEnabled()
+    {
+        $ask = $this->io->ask(Argument::cetera())->will(function (array $args) {
+            $message = array_shift($args);
+            return strpos($message, 'What is the amount of visits') === 0 ? 20 : 'asked';
+        });
+        $confirm = $this->io->confirm(Argument::cetera())->willReturn(true);
+
+        $config = new CustomizableAppConfig();
+
+        $this->plugin->process($this->io->reveal(), $config);
+
+        $this->assertTrue($config->hasApp());
+        $this->assertEquals([
+            'SECRET' => 'asked',
+            'DISABLE_TRACK_PARAM' => 'asked',
+            'CHECK_VISITS_THRESHOLD' => true,
+            'VISITS_THRESHOLD' => 20,
+        ], $config->getApp());
+        $ask->shouldHaveBeenCalledTimes(3);
+        $confirm->shouldHaveBeenCalledTimes(1);
     }
 
     /**
@@ -56,6 +89,8 @@ class ApplicationConfigCustomizerTest extends TestCase
         $config = new CustomizableAppConfig();
         $config->setApp([
             'SECRET' => 'foo',
+            'CHECK_VISITS_THRESHOLD' => true,
+            'VISITS_THRESHOLD' => 20,
         ]);
 
         $this->plugin->process($this->io->reveal(), $config);
@@ -63,6 +98,8 @@ class ApplicationConfigCustomizerTest extends TestCase
         $this->assertEquals([
             'SECRET' => 'foo',
             'DISABLE_TRACK_PARAM' => 'disable_param',
+            'CHECK_VISITS_THRESHOLD' => true,
+            'VISITS_THRESHOLD' => 20,
         ], $config->getApp());
         $ask->shouldHaveBeenCalledTimes(1);
     }
@@ -78,6 +115,8 @@ class ApplicationConfigCustomizerTest extends TestCase
         $config->setApp([
             'SECRET' => 'foo',
             'DISABLE_TRACK_PARAM' => 'the_new_secret',
+            'CHECK_VISITS_THRESHOLD' => true,
+            'VISITS_THRESHOLD' => 20,
         ]);
 
         $this->plugin->process($this->io->reveal(), $config);
@@ -85,7 +124,52 @@ class ApplicationConfigCustomizerTest extends TestCase
         $this->assertEquals([
             'SECRET' => 'foo',
             'DISABLE_TRACK_PARAM' => 'the_new_secret',
+            'CHECK_VISITS_THRESHOLD' => true,
+            'VISITS_THRESHOLD' => 20,
         ], $config->getApp());
         $ask->shouldNotHaveBeenCalled();
     }
+
+    /**
+     * @test
+     * @dataProvider provideInvalidValues
+     * @param mixed $value
+     */
+    public function validateVisitsThresholdThrowsExceptionWhenProvidedValueIsInvalid($value)
+    {
+        $this->expectException(InvalidConfigOptionException::class);
+        $this->plugin->validateVisitsThreshold($value);
+    }
+
+    public function provideInvalidValues(): array
+    {
+        return [
+            'string' => ['foo'],
+            'empty string' => [''],
+            'negative number' => [-5],
+            'negative number as string' => ['-5'],
+            'zero' => [0],
+            'zero as string' => ['0'],
+        ];
+    }
+
+    /**
+     * @test
+     * @dataProvider provideValidValues
+     * @param mixed $value
+     */
+    public function validateVisitsThresholdCastsToIntWhenProvidedValueIsValid($value, int $expected)
+    {
+        $this->assertEquals($expected, $this->plugin->validateVisitsThreshold($value));
+    }
+
+    public function provideValidValues(): array
+    {
+        return [
+            'positive as string' => ['20', 20],
+            'positive as integer' => [5, 5],
+            'one as string' => ['1', 1],
+            'one as integer' => [1, 1],
+        ];
+    }
 }