Added code coverage tool for pgAdmin. Fixes #5048.

docs/en_US/release_notes_4_17.rst

@@ -20,6 +20,7 @@ Housekeeping
 | `Issue #5023 <https://redmine.postgresql.org/issues/5023>`_ -  Refactored SQL of Views and Materialized Views.
 | `Issue #5024 <https://redmine.postgresql.org/issues/5024>`_ -  Refactored SQL of Functions and Procedures.
 | `Issue #5038 <https://redmine.postgresql.org/issues/5038>`_ -  Added support for on-demand loading of items in Select2.
+| `Issue #5048 <https://redmine.postgresql.org/issues/5048>`_ -  Added code coverage tool for pgAdmin.
 
 Bug fixes
 *********

web/regression/.coveragerc.in

@@ -0,0 +1,25 @@
+# .coveragerc to control coverage.py
+# This is default coverage configuration file require to run coverage.py
+# Any paths mentioned for coverage command are indented(default:4 spaces)
+
+[run]
+# Mention paths in 'source' to measure code coverage(Full project path)
+source =
+    $PGADMIN4_SRC/web/pgadmin/
+
+# Mention patterns in 'omit' to omit code from the coverage measurement
+# Here, omit code which contains pattern '/tests/', 'feature_tests' etc.
+omit =
+    */tests/*
+    */feature_tests/*
+
+# Mention patterns in 'include' to include code for the coverage measurement
+# Here, code coverage will show report only for modules mentioned in the
+# 'include'
+include =
+    # For all modules
+    */web/pgadmin/*
+    # For 'databases'
+    # */web/pgadmin/browser/server_groups/servers/databases/*
+    # For 'tables'
+    # */web/pgadmin/browser/server_groups/servers/databases/schemas/tables/*

web/regression/README

@@ -190,6 +190,41 @@ Python Tests:
      Example 2)  Execute only reverse engineered SQL test framework for some modules
          run 'python runtests.py --pkg resql --modules sequences,functions'
 
+Code Coverage:
+---------------
+
+- Test framework is able to calculate the code coverage.
+- Coverage package(coverage) is added in $PGADMIN4_SRC/web/regression/requirements.txt file
+
+How to generate code coverage report for API test-suite?
+-------------------------------------
+
+- Change to the regression test directory:
+    run 'cd $PGADMIN4_SRC/web/regression'
+
+- Before running code coverage we need to configure 'regression/.coveragerc' file.
+   i). Create 'regression/.coveragerc' file.
+   ii). Copy content of 'regression/.coveragerc.in' to 'regression/.coveragerc'
+   iii). Modify 'regression/.coveragerc' file as per our need as 'regression/.coveragerc.in' has default configurations
+
+        In 'regression/.coveragerc' file we need to mention some parameters
+        like 'source'(project path),'include'(files/modules to be included for
+        coverage),'omit'(files/modules to be omitted from coverage)
+
+        We can also add more parameters according to our need.
+        For more info please read coverage.py's official document here
+        'http://coverage.readthedocs.io/en/coverage-4.2/install.html'
+
+- Run coverage
+    With all modules
+        run 'python runtests.py --coverage --exclude feature_tests'
+    With specific module
+        run 'python runtests.py --pkg browser.server_groups.servers.tests --coverage'
+
+- After execution of coverage, we will see code coverage report on console.
+  For a nicer presentation, '/regression/covhtml' directory gets created.
+  Open 'index.html' file in browser and you will see detail coverage report.
+
 Javascript Tests:
 
 - Install node.js and yarn, in the appropriate way for your platform. On macOS with Homebrew you might use:

web/regression/python_test_utils/test_utils.py

@@ -14,6 +14,7 @@ import sys
 import uuid
 import psycopg2
 import sqlite3
+import shutil
 from functools import partial
 from testtools.testcase import clone_test_with_new_id
 
@@ -23,7 +24,11 @@ from regression import test_setup
 
 from pgadmin.utils.preferences import Preferences
 
+CURRENT_PATH = os.path.abspath(os.path.join(os.path.dirname(
+    os.path.realpath(__file__)), "../"))
+
 SERVER_GROUP = test_setup.config_data['server_group']
+COVERAGE_CONFIG_FILE = os.path.join(CURRENT_PATH, ".coveragerc")
 file_name = os.path.realpath(__file__)
 
 
@@ -1146,3 +1151,27 @@ def get_watcher_dialogue_status(self):
 def get_driver_version():
     version = getattr(psycopg2, '__version__', None)
     return version
+
+
+def is_coverage_enabled(args):
+    """
+    This function checks for coverage args exists in command line args
+    :return: boolean
+    """
+    if "coverage" in args and args["coverage"]:
+        return True
+    return False
+
+
+def print_and_store_coverage_report(cov):
+    """
+    This function print the coverage report on console and store it in html
+    files
+    :return: None
+    """
+    print("\nCoverage Summary:\n", file=sys.stderr)
+    cov.report()
+    cov_dir = os.path.join(CURRENT_PATH, "covhtml")
+    if os.path.exists(cov_dir):
+        shutil.rmtree(cov_dir)
+    cov.html_report(directory=cov_dir)

web/regression/requirements.txt

@@ -27,7 +27,7 @@ testscenarios==0.5.0
 testtools==2.3.0
 traceback2==1.4.0
 selenium==3.14.0
-
+coverage==5.0.1
 ###############################################################
 # Modules specifically required for Python3.3 or lesser version
 ###############################################################

web/regression/runtests.py

@@ -20,6 +20,7 @@ import sys
 import traceback
 import json
 import random
+import coverage
 
 import unittest
 
@@ -48,6 +49,8 @@ if sys.path[0] != root:
 from pgadmin import create_app
 import config
 
+COVERAGE_CONFIG_FILE = os.path.join(CURRENT_PATH, ".coveragerc")
+
 if config.SERVER_MODE is True:
     config.SECURITY_RECOVERABLE = True
     config.SECURITY_CHANGEABLE = True
@@ -297,6 +300,8 @@ def add_arguments():
         help='Skips execution of the test cases of particular package and '
              'sub-packages'
     )
+    parser.add_argument('--coverage', nargs='?', const=True, type=bool,
+                        default=False, help='Enable code coverage feature')
     parser.add_argument(
         '--default_browser',
         help='Executes the feature test in specific browser'
@@ -394,6 +399,7 @@ if __name__ == '__main__':
     # Failure detected?
     failure = False
     test_result = dict()
+    cov = None
 
     # Set signal handler for cleanup
     signal_list = dir(signal)
@@ -430,6 +436,12 @@ if __name__ == '__main__':
     node_name = "all"
     if args['pkg'] is not None:
         node_name = args['pkg'].split('.')[-1]
+
+    # Start coverage
+    if test_utils.is_coverage_enabled(args):
+        cov = coverage.Coverage(config_file=COVERAGE_CONFIG_FILE)
+        cov.start()
+
     try:
         for server in servers_info:
             print("\n=============Running the test cases for '%s'============="
@@ -557,6 +569,15 @@ if __name__ == '__main__':
         file=sys.stderr
     )
 
+    # Stop code coverage
+    if test_utils.is_coverage_enabled(args):
+        cov.stop()
+        cov.save()
+
+    # # Print coverage only if coverage args given in command line
+    if test_utils.is_coverage_enabled(args):
+        test_utils.print_and_store_coverage_report(cov)
+
     print("Please check output in file: %s/regression.log\n" % CURRENT_PATH)
 
     # Unset environment variable