diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 95cab5132f..85669b80ee 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2204,7 +2204,9 @@ class RunQueueExecute:
                     if nexttask in self.sqdata.outrightfail:
                         logger.debug2('No package found, so skipping setscene task %s', nexttask)
                         self.sq_task_failoutright(nexttask)
-                        return True
+                        # Technically we should return True here but that would then start the task list back at the start
+                        # which is really bad for performance ("time bitbake world -n"). Instead, keep our place in the tasklist.
+                        continue
                     if nexttask in self.sqdata.unskippable:
                         logger.debug2("Setscene task %s is unskippable" % nexttask)
                     task = nexttask