diff --git a/libhts/libhts.py b/libhts/libhts.py
index bb4cdda60c700f456c0618bb4d664bb441b797a6..a68c2d622ed87b38d6c9fe23e330275ce59adb49 100644
--- a/libhts/libhts.py
+++ b/libhts/libhts.py
@@ -456,7 +456,17 @@ def size_factor_correlations(counts_data, summaries, normalizer):
     # by_norm = counts_data / size_factors
 
     def compute_pearsonr_with_size_factor(row):
-        return pearsonr(row, size_factors)[0]
+        try:
+            return pearsonr(row, size_factors)[0]
+        except ValueError as err:
+            if str(err) == "array must not contain infs or NaNs":
+                msg = (
+                    f"Cannot compute Pearson correlation with size factors "
+                    f"when normalizing using {normaliser}")
+                warnings.warn(msg)
+                return np.nan
+            else:
+                raise
     # return by_norm.apply(compute_pearsonr_with_size_factor, axis=1)
     return (counts_data / size_factors).apply(compute_pearsonr_with_size_factor, axis=1)