API doc: validate generated reference

Extend 'makeapi --validate' to validate API Reference files too. If
differences are found between the generated and stored docs the
validation fails. This command is executed in our Azure pipelines, so
every time a developer opens a PR but forgets to update the API
Reference, the CI will fail.

Fixes: https://pagure.io/freeipa/issue/9287
Signed-off-by: Antonio Torres <antorres@redhat.com>
Reviewed-By: Rob Crittenden <rcritten@redhat.com>

makeapi.in

@@ -268,7 +268,8 @@ def make_api():
 
     return 0
 
-def make_api_reference():
+
+def make_api_reference(validate_only=False):
     """
     Generate API reference.
     Index will be created in reStructuredText format so that it can be
@@ -324,6 +325,8 @@ def make_api_reference():
             ret.extend(all_subclasses(s))
         return sorted(list(set(ret)), key=operator.attrgetter('__name__'))
 
+    api_contents = {}
+
     ipa_classes = all_subclasses(Param)
 
     for param in ipa_classes:
@@ -331,11 +334,11 @@ def make_api_reference():
             ".. _%s:\n" % param.__name__,
             "# %s" % param.__name__,
         ]
-        with open("doc/api/%s.md" % param.__name__, "w") as f:
-            out = "\n".join(lines)
-            f.write(out)
+        api_contents["doc/api/%s.md" % param.__name__] = "\n".join(lines)
         param_lines.append("   %s.md" % param.__name__)
 
+    api_contents["doc/api/parameters.rst"] = "\n".join(param_lines)
+
     def generate_param_type_text(obj):
         # If class is part of IPA Params, return text with ref, if not just return its name
         if type(obj) in ipa_classes:
@@ -393,16 +396,20 @@ def make_api_reference():
         except FileNotFoundError:
                 notes = notes_template
 
-        with open("doc/api/%s.md" % cmd.name, "w") as f:
-            out = class_template.format(
+        out = class_template.format(
                 reference="\n".join(lines), notes=notes).strip()
-            f.write(out)
+        api_contents["doc/api/%s.md" % cmd.name] = out
 
-    with open("doc/api/commands.rst", "w") as f:
-        f.write("\n".join(cmd_lines))
+    api_contents["doc/api/commands.rst"] = "\n".join(cmd_lines)
 
-    with open("doc/api/parameters.rst", "w") as f:
-        f.write("\n".join(param_lines))
+    if validate_only:
+        # Don't write to files, just return contents to be validated
+        return api_contents
+    else:
+        # Write all contents to their files
+        for file, contents in api_contents.items():
+            with open(file, "w") as f:
+                f.write(contents)
 
     return 0
 
@@ -633,6 +640,26 @@ def validate_api():
 
     return rval
 
+
+def validate_api_reference():
+    api_contents = make_api_reference(validate_only=True)
+    rval = 0
+
+    # Iterate current files, fail if any diference is found
+    for file, contents in api_contents.items():
+        try:
+            f = open(file, "r")
+            file_contents = f.read().split("//end)")[0].strip()
+            api_contents = contents.split("//end")[0].strip()
+            if file_contents != api_contents:
+                rval = 1
+                print("API Reference difference found in: ", file)
+        except FileNotFoundError:
+            rval = 1
+            print("File not found: ", file)
+
+    return rval
+
 def main():
     rval = 0
     options, _args = parse_options()
@@ -662,6 +689,7 @@ def main():
             rval |= API_NO_FILE
         else:
             rval |= validate_api()
+            rval |= validate_api_reference()
     else:
         print("Writing API to API.txt")
         rval |= make_api()
@@ -670,7 +698,9 @@ def main():
 
     if rval & API_FILE_DIFFERENCE:
         print('')
-        print('There are one or more changes to the API.\nEither undo the API changes or update API.txt and increment the major version in VERSION.')
+        print("There are one or more changes to the API.\n"
+              "Either undo the API changes or update API.txt, "
+              "API Reference, and increment the major version in VERSION.")
 
     if rval & API_NEW_COMMAND:
         print('')