From 0c1010d6f7585095402f47c111b8bb6cde3068b3 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 10 Jul 2018 15:49:38 +0200 Subject: [PATCH] Mark all expected failures as strict With strict=True, xfail() fails when the test case passes unexpectably. This allows us to spot passing tests that are expected to fail. Fixes: https://pagure.io/freeipa/issue/7613 Signed-off-by: Christian Heimes Reviewed-By: Petr Cech --- ipatests/test_integration/test_caless.py | 16 ++++++++-------- ipatests/test_integration/test_installation.py | 8 ++++---- .../test_integration/test_replica_promotion.py | 10 +++++----- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ipatests/test_integration/test_caless.py b/ipatests/test_integration/test_caless.py index 89466fc93..09e2c4ec1 100644 --- a/ipatests/test_integration/test_caless.py +++ b/ipatests/test_integration/test_caless.py @@ -455,7 +455,7 @@ class TestServerInstall(CALessBase): 'The full certificate chain is not present in ' '%s/server.p12' % self.master.config.test_dir) - @pytest.mark.xfail(reason='Ticket N 6289') + @pytest.mark.xfail(reason='Ticket N 6289', strict=True) @server_install_teardown def test_ca_2_certs(self): "IPA server install with CA PEM file with 2 certificates" @@ -518,7 +518,7 @@ class TestServerInstall(CALessBase): 'ipa-server-install: error: You must specify ' '--dirsrv-pin with --dirsrv-cert-file') - @pytest.mark.xfail(reason='freeipa ticket 5378') + @pytest.mark.xfail(reason='freeipa ticket 5378', strict=True) @server_install_teardown def test_incorect_http_pin(self): "IPA server install with incorrect HTTP PKCS#12 password" @@ -529,7 +529,7 @@ class TestServerInstall(CALessBase): result = self.install_server(http_pin='bad') assert_error(result, 'incorrect password for pkcs#12 file server.p12') - @pytest.mark.xfail(reason='freeipa ticket 5378') + @pytest.mark.xfail(reason='freeipa ticket 5378', strict=True) @server_install_teardown def test_incorect_ds_pin(self): "IPA server install with incorrect DS PKCS#12 password" @@ -872,7 +872,7 @@ class TestReplicaInstall(CALessBase): assert_error(result, 'Failed to open %s/does_not_exist' % self.master.config.test_dir) - @pytest.mark.xfail(reason='freeipa ticket 5378') + @pytest.mark.xfail(reason='freeipa ticket 5378', strict=True) @replica_install_teardown def test_incorect_http_pin(self): "IPA replica install with incorrect HTTP PKCS#12 password" @@ -883,7 +883,7 @@ class TestReplicaInstall(CALessBase): assert result.returncode > 0 assert_error(result, 'incorrect password for pkcs#12 file replica.p12') - @pytest.mark.xfail(reason='freeipa ticket 5378') + @pytest.mark.xfail(reason='freeipa ticket 5378', strict=True) @replica_install_teardown def test_incorect_ds_pin(self): "IPA replica install with incorrect DS PKCS#12 password" @@ -1379,7 +1379,7 @@ class TestCertInstall(CALessBase): cert_exists=False) assert_error(result, 'Failed to open does_not_exist') - @pytest.mark.xfail(reason='freeipa ticket 5378') + @pytest.mark.xfail(reason='freeipa ticket 5378', strict=True) def test_incorect_http_pin(self): "Install new HTTP certificate with incorrect PKCS#12 password" @@ -1387,7 +1387,7 @@ class TestCertInstall(CALessBase): assert_error(result, 'incorrect password for pkcs#12 file server.p12') - @pytest.mark.xfail(reason='freeipa ticket 5378') + @pytest.mark.xfail(reason='freeipa ticket 5378', strict=True) def test_incorect_dirsrv_pin(self): "Install new DS certificate with incorrect PKCS#12 password" @@ -1473,7 +1473,7 @@ class TestCertInstall(CALessBase): result = self.certinstall('w', 'ca1/subca/server') assert result.returncode == 0, result.stderr_text - @pytest.mark.xfail(reason='freeipa ticket 6959') + @pytest.mark.xfail(reason='freeipa ticket 6959', strict=True) def test_ds_intermediate_ca(self): "Install new DS certificate issued by intermediate CA" diff --git a/ipatests/test_integration/test_installation.py b/ipatests/test_integration/test_installation.py index 8a24e32ff..d42243bed 100644 --- a/ipatests/test_integration/test_installation.py +++ b/ipatests/test_integration/test_installation.py @@ -152,7 +152,7 @@ class ADTrustInstallTestBase(IntegrationTest): # Master X Replicas installation tests ## -@pytest.mark.xfail(reason="FreeIPA ticket 7008") +@pytest.mark.xfail(reason="FreeIPA ticket 7008", strict=True) class TestInstallWithCA1(InstallTestBase1): @classmethod @@ -175,7 +175,7 @@ class TestInstallWithCA1(InstallTestBase1): super(TestInstallWithCA1, self).test_replica2_ipa_dns_install() -@pytest.mark.xfail(reason="FreeIPA ticket 7008") +@pytest.mark.xfail(reason="FreeIPA ticket 7008", strict=True) class TestInstallWithCA2(InstallTestBase2): @classmethod @@ -215,7 +215,7 @@ class TestInstallWithCA_KRA2(InstallTestBase2): tasks.install_master(cls.master, setup_dns=False, setup_kra=True) -@pytest.mark.xfail(reason="FreeIPA ticket 7008") +@pytest.mark.xfail(reason="FreeIPA ticket 7008", strict=True) class TestInstallWithCA_DNS1(InstallTestBase1): @classmethod @@ -238,7 +238,7 @@ class TestInstallWithCA_DNS1(InstallTestBase1): super(TestInstallWithCA_DNS1, self).test_replica2_ipa_dns_install() -@pytest.mark.xfail(reason="FreeIPA ticket 7008") +@pytest.mark.xfail(reason="FreeIPA ticket 7008", strict=True) class TestInstallWithCA_DNS2(InstallTestBase2): @classmethod diff --git a/ipatests/test_integration/test_replica_promotion.py b/ipatests/test_integration/test_replica_promotion.py index a0da430e0..d5ef49a7d 100644 --- a/ipatests/test_integration/test_replica_promotion.py +++ b/ipatests/test_integration/test_replica_promotion.py @@ -64,7 +64,7 @@ class TestReplicaPromotionLevel0(ReplicaPromotionBase): 'You must provide a file generated by ipa-replica-prepare' ' to create a replica when the domain is at level 0', 1) - @pytest.mark.xfail(reason="Ticket N 6274") + @pytest.mark.xfail(reason="Ticket N 6274", strict=True) @replicas_cleanup def test_backup_restore(self): """ @@ -87,7 +87,7 @@ class TestReplicaPromotionLevel0(ReplicaPromotionBase): assert(found2 > 0), result2.stdout_text -@pytest.mark.xfail(reason="Ticket N 6274") +@pytest.mark.xfail(reason="Ticket N 6274", strict=True) class TestKRAInstall(IntegrationTest): """ TestCase: http://www.freeipa.org/page/V4/Replica_Promotion/Test_plan @@ -126,7 +126,7 @@ class TestKRAInstall(IntegrationTest): tasks.install_kra(replica2) -@pytest.mark.xfail(reason="Ticket N 6274") +@pytest.mark.xfail(reason="Ticket N 6274", strict=True) class TestCAInstall(IntegrationTest): topology = 'star' domain_level = DOMAIN_LEVEL_0 @@ -205,7 +205,7 @@ class TestReplicaPromotionLevel1(ReplicaPromotionBase): '-U']) -@pytest.mark.xfail(reason="Ticket N 6274") +@pytest.mark.xfail(reason="Ticket N 6274", strict=True) class TestReplicaManageCommands(IntegrationTest): topology = "star" num_replicas = 2 @@ -372,7 +372,7 @@ class TestProhibitReplicaUninstallation(IntegrationTest): '-U', '--ignore-topology-disconnect']) -@pytest.mark.xfail(reason="Ticket N 6274") +@pytest.mark.xfail(reason="Ticket N 6274", strict=True) class TestOldReplicaWorksAfterDomainUpgrade(IntegrationTest): topology = 'star' num_replicas = 1