diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 2a9fb662..3668a32b 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -28,6 +28,7 @@ import datetime
 import pickle
 import traceback
 import gc
+import stat
 import bb.server.xmlrpcserver
 from bb import daemonize
 from multiprocessing import queues
@@ -64,6 +65,9 @@ class ProcessServer():
         self.bitbake_lock_name = lockname
         self.sock = sock
         self.sockname = sockname
+        # It is possible the directory may be renamed. Cache the inode of the socket file
+        # so we can tell if things changed.
+        self.sockinode = os.stat(self.sockname)[stat.ST_INO]
 
         self.server_timeout = server_timeout
         self.timeout = self.server_timeout
@@ -246,8 +250,14 @@ class ProcessServer():
 
         serverlog("Exiting")
         # Remove the socket file so we don't get any more connections to avoid races
+        # The build directory could have been renamed so if the file isn't the one we created
+        # we shouldn't delete it.
         try:
-            os.unlink(self.sockname)
+            sockinode = os.stat(self.sockname)[stat.ST_INO]
+            if sockinode == self.sockinode:
+                os.unlink(self.sockname)
+            else:
+                serverlog("bitbake.sock inode mismatch (%s vs %s), not deleting." % (sockinode, self.sockinode))
         except Exception as err:
             serverlog("Removing socket file '%s' failed (%s)" % (self.sockname, err))
         self.sock.close()