arch/x86/include/asm/pgtable-3level.h | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index 43876f16caf1..4d6bf9f54bad 100644
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -53,33 +53,34 @@ static inline void native_set_pte(pte_t *ptep, pte_t pte)
  *
  * With THP if the mmap_sem is hold for reading, the pmd can become
  * THP or null or point to a pte (and in turn become "stable") at any
- * time under pmd_read_atomic, so it's mandatory to read it atomically
- * with cmpxchg8b.
+ * time under pmd_read_atomic, so we have to be really careful. We'll
+ * re-read the low word to check that it hasn't become NULL or turned
+ * into a pte.
  */
-#ifndef CONFIG_TRANSPARENT_HUGEPAGE
 static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
 {
 	pmdval_t ret;
-	u32 *tmp = (u32 *)pmdp;
+	u32 low, hight, *tmp = (u32 *)pmdp;
 
-	ret = (pmdval_t) (*tmp);
-	if (ret) {
+repeat:
+	low = tmp[0];
+	high = 0;
+	if (low) {
 		/*
 		 * If the low part is null, we must not read the high part
 		 * or we can end up with a partial pmd.
 		 */
 		smp_rmb();
-		ret |= ((pmdval_t)*(tmp + 1)) << 32;
+		high = tmp[1];
+		if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
+			smp_rmb();
+			if (low != tmp[0])
+				goto repeat;
+		}
 	}
 
-	return (pmd_t) { ret };
+	return (pmd_t) { low + ((u64)high << 32) };
 }
-#else /* CONFIG_TRANSPARENT_HUGEPAGE */
-static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
-{
-	return (pmd_t) { atomic64_read((atomic64_t *)pmdp) };
-}
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
 {