diff --git a/meta/lib/oe/package_manager/rpm/__init__.py b/meta/lib/oe/package_manager/rpm/__init__.py
index b392581069..dce5912329 100644
--- a/meta/lib/oe/package_manager/rpm/__init__.py
+++ b/meta/lib/oe/package_manager/rpm/__init__.py
@@ -80,7 +80,9 @@ class RpmPM(PackageManager):
         self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name)
         if not os.path.exists(self.d.expand('${T}/saved_packaging_data')):
             bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data'))
-        self.packaging_data_dirs = ['etc/rpm', 'etc/rpmrc', 'etc/dnf', 'var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf']
+        self.packaging_data_rpm_dirs = ['etc/rpm', 'etc/rpmrc', 'var/lib/rpm']
+        self.packaging_data_dnf_dirs = ['etc/dnf', 'var/lib/dnf', 'var/cache/dnf']
+        self.packaging_data_dirs = self.packaging_data_rpm_dirs + self.packaging_data_dnf_dirs
         self.solution_manifest = self.d.expand('${T}/saved/%s_solution' %
                                                self.task_name)
         if not os.path.exists(self.d.expand('${T}/saved')):
@@ -237,8 +239,11 @@ class RpmPM(PackageManager):
         self._invoke_dnf(["autoremove"])
 
     def remove_packaging_data(self):
+        remove_packaging_data_dirs = self.packaging_data_dnf_dirs
+        if "rpm" not in self.list_installed():
+            remove_packaging_data_dirs += self.packaging_data_rpm_dirs
         self._invoke_dnf(["clean", "all"])
-        for dir in self.packaging_data_dirs:
+        for dir in remove_packaging_data_dirs:
             bb.utils.remove(oe.path.join(self.target_rootfs, dir), True)
 
     def backup_packaging_data(self):