diff --git a/CHANGES b/CHANGES
index b860e53ed..d28d1cf7a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -33,6 +33,7 @@ Bugs fixed
 * #5091: latex: curly braces in index entries are not handled correctly
 * #5070: epub: Wrong internal href fragment links
 * #5104: apidoc: Interface of ``sphinx.apidoc:main()`` has changed
+* #4272: PDF builds of French projects have issues with XeTeX
 
 Testing
 --------
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index e37e6f395..a763e7c15 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -453,6 +453,14 @@ class LaTeXTranslator(nodes.NodeVisitor):
         # sort out some elements
         self.elements = DEFAULT_SETTINGS.copy()
         self.elements.update(ADDITIONAL_SETTINGS.get(builder.config.latex_engine, {}))
+        # for xelatex+French, don't use polyglossia
+        if self.elements['latex_engine'] == 'xelatex':
+            if builder.config.language:
+                if builder.config.language[:2] == 'fr':
+                    self.elements.update({
+                        'polyglossia': '',
+                        'babel':       '\\usepackage{babel}',
+                    })
         # allow the user to override them all
         self.check_latex_elements()
         self.elements.update(builder.config.latex_elements)