diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 102927db4f2a..2a55cd63abeb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,12 +18,15 @@ stages:
     BB_LOGCONFIG: $CI_PROJECT_DIR/ci/logging.yml
     TOOLCHAIN_DIR: $CI_BUILDS_DIR/persist/toolchains
     IMAGE_DIR: $CI_PROJECT_DIR/work/build/tmp/deploy/images
+    TOOLCHAIN_LINK_DIR: $CI_PROJECT_DIR/work/build/toolchains
   before_script:
     - echo KAS_WORK_DIR = $KAS_WORK_DIR
     - echo SSTATE_DIR = $SSTATE_DIR
     - echo DL_DIR = $DL_DIR
     - rm -rf $KAS_WORK_DIR
-    - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR $DL_DIR $TOOLCHAIN_DIR
+    - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR $DL_DIR $TOOLCHAIN_DIR $TOOLCHAIN_LINK_DIR
+    # Must do this here, as it's the only way to make sure the toolchain is installed on the same builder
+    - ./ci/get-binary-toolchains $DL_DIR $TOOLCHAIN_DIR $TOOLCHAIN_LINK_DIR
 
 # Generalised fragment to do a Kas build
 .build:
@@ -43,13 +46,6 @@ update-repos:
   script:
   - flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos
 
-get-binary-toolchains:
-  extends: .setup
-  stage: prep
-  script:
-    - ./ci/get-binary-toolchains $DL_DIR $TOOLCHAIN_DIR
-
-
 #
 # Bootstrap stage, bootstrap and machine coverage
 #
diff --git a/ci/external-gccarm.yml b/ci/external-gccarm.yml
index ad0329835041..0ce47952ba3f 100644
--- a/ci/external-gccarm.yml
+++ b/ci/external-gccarm.yml
@@ -5,4 +5,4 @@ local_conf_header:
   cc: |
     PNBLACKLIST[gcc-cross-arm] = "Using external toolchain"
     TCMODE = "external-arm"
-    EXTERNAL_TOOLCHAIN = "${TOOLCHAIN_DIR}/${TARGET_ARCH}"
+    EXTERNAL_TOOLCHAIN = "${TOPDIR}/toolchains/${TARGET_ARCH}"
diff --git a/ci/get-binary-toolchains b/ci/get-binary-toolchains
index 6fa7ff439232..e860ec267e18 100755
--- a/ci/get-binary-toolchains
+++ b/ci/get-binary-toolchains
@@ -6,9 +6,10 @@ VER="10.2-2020.11"
 
 DOWNLOAD_DIR=$1
 TOOLCHAIN_DIR=$2
+TOOLCHAIN_LINK_DIR=$3
 
-# These should be already created by .bitlab-ci.yml, but do here if run outside of that env
-mkdir -p $DOWNLOAD_DIR $TOOLCHAIN_DIR
+# These should be already created by .gitlab-ci.yml, but do here if run outside of that env
+mkdir -p $DOWNLOAD_DIR $TOOLCHAIN_DIR $TOOLCHAIN_LINK_DIR
 
 if [ $HOST_ARCH = "aarch64" ]; then
 	#AArch64 Linux hosted cross compilers
@@ -32,21 +33,14 @@ else
 fi
 
 for i in arm aarch64 aarch64_be; do
-	if [ ! -f $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz ]; then
-		continue
-	fi
-
-	if [ -d $TOOLCHAIN_DIR/$i ]; then
-		echo "$TOOLCHAIN_DIR/$i EXISTS!"
-		MANIFEST=$(ls $TOOLCHAIN_DIR/$i | grep txt)
-		if [[ $MANIFEST != $VER-$HOST_ARCH-$i-none-linux-gnu*.txt ]]; then
-			echo "Removing old $MANIFEST for $VER-$HOST_ARCH-$i-*.txt toolchain"
-			rm -rf $TOOLCHAIN_DIR/$i
+	if [ ! -d $TOOLCHAIN_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*/ ]; then
+		if [ ! -f $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz ]; then
+			continue
 		fi
-	fi
 
-	if [ ! -d $TOOLCHAIN_DIR/$i ]; then
 		tar -C $TOOLCHAIN_DIR -axvf $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz
-		mv $TOOLCHAIN_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*/ $TOOLCHAIN_DIR/$i
 	fi
+
+	# Setup a link for the toolchain to use local to the building machine (e.g., not in a shared location)
+	ln -s $TOOLCHAIN_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu* $TOOLCHAIN_LINK_DIR/$i
 done