[RFC,v2] cargo_common.bbclass: use source replacement instead of dependency patching

diff --git a/meta/classes-recipe/cargo_common.bbclass b/meta/classes-recipe/cargo_common.bbclass
index c9eb2d09a5..6056248cfd 100644
--- a/meta/classes-recipe/cargo_common.bbclass
+++ b/meta/classes-recipe/cargo_common.bbclass
@@ -127,8 +127,112 @@  cargo_common_do_configure () {
 }
 
 python cargo_common_do_patch_paths() {
+    import glob
     import shutil
 
+    def is_workspace_or_package(f, workspace_or_package):
+        """
+        Checks if the manifest contains [workspace] or [package] header
+        f is a file-like cargo manifest, workspace_or_package is a string,
+        either 'workspace' or 'package'
+        """
+
+        f.seek(0)
+        for line in f.readlines():
+            if line.strip() == '[%s]' % workspace_or_package:
+                return True
+        return False
+
+    def is_package(f):
+        return is_workspace_or_package(f, 'package')
+
+    def is_workspace(f):
+        return is_workspace_or_package(f, 'workspace')
+
+    def extract_package_name(f):
+        """Extract crate name"""
+        f.seek(0)
+        found_package_header = False
+        for line in f.readlines():
+            if not found_package_header and line.strip() == '[package]':
+                found_package_header = True
+                continue
+            if found_package_header and line.strip().startswith('name ='):
+                return line.split('=')[1].strip().strip('"')
+
+    def extract_ws_members(f):
+        """
+        Find the [workspace] header in the file, and then find the 'members' key. Then
+        extract all the elements from the corresponding array, unless it's a comment.
+        f should a file-like object of the Cargo manifest.
+        """
+
+        import re
+        f.seek(0)
+        found_ws_header = False
+        found_members_start = False
+        ws_members = []
+        # regex to extract something from between two double-quotes
+        member_regex = r'\"([^\"]+)\"'
+
+        for line in f.readlines():
+            if not found_ws_header and line.strip() == '[workspace]':
+                found_ws_header = True
+                continue
+            if found_ws_header and not found_members_start and line.strip().startswith('members'):
+                found_members_start = True
+            if found_members_start and not line.strip().startswith('#'):
+                matches = re.findall(member_regex, line)
+                ws_members.extend(matches)
+                # the end of the member list
+                if ']' in line:
+                    break
+        bb.note("Extracted workspace members: %s" % ws_members)
+        return ws_members
+
+    def extract_git_repos_from_lockfile(f):
+        """Return a list of repositories that start with 'git+' protocol from Cargo.lock"""
+        f.seek(0)
+        repos = []
+        for line in f.readlines():
+            if line.startswith('source = "git+'):
+                repos.append(line.split('"')[1])
+        bb.note("Extracted git repos: %s" % repos)
+        return repos
+
+    def is_rust_crate_folder(path):
+        cargo_toml_path = os.path.join(path, 'Cargo.toml')
+        return os.path.exists(cargo_toml_path)
+
+    def get_matching_repo_from_lockfile(lockfile_repos, repo, revision):
+        for lf_repo in lockfile_repos.keys():
+            if repo in lf_repo and lf_repo.endswith(revision):
+                lockfile_repos[lf_repo] = True
+                return lf_repo.split("#")[0]
+        bb.fatal('Cannot find %s (%s) repository from SRC_URI in Cargo.lock file' % (repo, revision))
+
+    def create_cargo_checksum(folder_path):
+        bb.note("Creating cargo checksum for %s" % folder_path)
+        checksum_path = os.path.join(folder_path, '.cargo-checksum.json')
+        if os.path.exists(checksum_path):
+            return
+
+        import hashlib, json
+
+        checksum = {'files': {}}
+        for root, _, files in os.walk(folder_path):
+            for f in files:
+                full_path = os.path.join(root, f)
+                relative_path = os.path.relpath(full_path, folder_path)
+                if relative_path.startswith(".git/"):
+                    continue
+                with open(full_path, 'rb') as f2:
+                    file_sha = hashlib.sha256(f2.read()).hexdigest()
+                checksum["files"][relative_path] = file_sha
+
+        with open(checksum_path, 'w') as f:
+            json.dump(checksum, f)
+
     cargo_config = os.path.join(d.getVar("CARGO_HOME"), "config.toml")
     if not os.path.exists(cargo_config):
         return
@@ -137,66 +241,98 @@  python cargo_common_do_patch_paths() {
     if len(src_uri) == 0:
         return
 
-    patches = dict()
+    sources = dict()
     workdir = d.getVar('UNPACKDIR')
     fetcher = bb.fetch2.Fetch(src_uri, d)
-    for url in fetcher.urls:
-        ud = fetcher.ud[url]
-        if ud.type == 'git' or ud.type == 'gitsm':
-            name = ud.parm.get('name')
-            destsuffix = ud.parm.get('destsuffix')
-            if name is not None and destsuffix is not None:
-                if ud.user:
-                    repo = '%s://%s@%s%s' % (ud.proto, ud.user, ud.host, ud.path)
-                else:
-                    repo = '%s://%s%s' % (ud.proto, ud.host, ud.path)
-                path = '%s = { path = "%s" }' % (name, os.path.join(workdir, destsuffix))
-                patches.setdefault(repo, []).append(path)
 
-    with open(cargo_config, "a+") as config:
-        for k, v in patches.items():
-            print('\n[patch."%s"]' % k, file=config)
-            for name in v:
-                print(name, file=config)
+    vendor_folder = os.path.join(workdir, 'yocto-vendored-source-crates')
 
-    if not patches:
-        return
+    os.makedirs(vendor_folder, exist_ok=True)
 
-    # Cargo.lock file is needed for to be sure that artifacts
-    # downloaded by the fetch steps are those expected by the
-    # project and that the possible patches are correctly applied.
-    # Moreover since we do not want any modification
-    # of this file (for reproducibility purpose), we prevent it by
-    # using --frozen flag (in CARGO_BUILD_FLAGS) and raise a clear error
-    # here is better than letting cargo tell (in case the file is missing)
-    # "Cargo.lock should be modified but --frozen was given"
+    for url in fetcher.urls:
+        ud = fetcher.ud[url]
+        if ud.type != 'git' and ud.type != 'gitsm':
+            continue
+
+        destsuffix = ud.parm.get('destsuffix')
+        crate_folder = os.path.join(workdir, destsuffix)
+
+        if destsuffix is None or not is_rust_crate_folder(crate_folder):
+            continue
+
+        if ud.user:
+            repo = '%s://%s@%s%s' % (ud.proto, ud.user, ud.host, ud.path)
+        else:
+            repo = '%s://%s%s' % (ud.proto, ud.host, ud.path)
+
+        sources[destsuffix] = (repo, ud.revision, crate_folder)
+
+        cargo_toml_path = os.path.join(workdir, destsuffix, 'Cargo.toml')
+        cargo_toml_file = open(cargo_toml_path, 'r')
+
+        if is_workspace(cargo_toml_file):
+            bb.note("%s is a workspace" % cargo_toml_path)
+            members = extract_ws_members(cargo_toml_file)
+            for member in members:
+                # "member" is a relative path, and can contain wildcards
+                unglobbed_member_crate_folder = os.path.join(workdir, destsuffix, member)
+                for member_crate_folder in glob.glob(unglobbed_member_crate_folder):
+                    bb.note("Processing member: %s" % member_crate_folder)
+                    if not is_rust_crate_folder(member_crate_folder):
+                        bb.note("%s has no Cargo.toml, skipping" % member_crate_folder)
+                        continue
+
+                    member_crate_cargo_toml = os.path.join(member_crate_folder, 'Cargo.toml')
+                    with open(member_crate_cargo_toml) as mcct:
+                        member_crate_name = extract_package_name(mcct)
+                    shutil.copytree(member_crate_folder, os.path.join(vendor_folder, member_crate_name))
+
+        if is_package(cargo_toml_file):
+            bb.note("%s is a package" % cargo_toml_path)
+            crate_folder = os.path.join(workdir, destsuffix)
+            crate_name = extract_package_name(cargo_toml_file)
+            shutil.copytree(crate_folder, os.path.join(vendor_folder, crate_name))
+
+        cargo_toml_file.close()
+
+    if len(sources) == 0:
+        return
 
     lockfile = d.getVar("CARGO_LOCK_PATH")
     if not os.path.exists(lockfile):
         bb.fatal(f"{lockfile} file doesn't exist")
 
-    # There are patched files and so Cargo.lock should be modified but we use
-    # --frozen so let's handle that modifications here.
-    #
-    # Note that a "better" (more elegant ?) would have been to use cargo update for
-    # patched packages:
-    #  cargo update --offline -p package_1 -p package_2
-    # But this is not possible since it requires that cargo local git db
-    # to be populated and this is not the case as we fetch git repo ourself.
-
-    lockfile_orig = lockfile + ".orig"
-    if not os.path.exists(lockfile_orig):
-        shutil.copy(lockfile, lockfile_orig)
-
-    newlines = []
-    with open(lockfile_orig, "r") as f:
-        for line in f.readlines():
-            if not line.startswith("source = \"git"):
-                newlines.append(line)
+    # key is the repo url, value is a boolean, which is used later
+    # to indicate if there is a matching repository in SRC_URI also
+    lockfile_git_repos = {}
+    with open(lockfile, 'r') as lf:
+        lockfile_git_repos = {repo_url: False for repo_url in extract_git_repos_from_lockfile(lf)}
+
+    for d in os.scandir(vendor_folder):
+        if d.is_dir():
+            create_cargo_checksum(d.path)
+
+
+    with open(cargo_config, "a+") as config:
+        print('\n[source."yocto-vendored-sources"]', file=config)
+        print('directory = "%s"' % vendor_folder, file=config)
+
+        for destsuffix, (repo, revision, repo_path) in sources.items():
+            lockfile_repo = get_matching_repo_from_lockfile(lockfile_git_repos, repo, revision)
+            print('\n[source."%s"]' % lockfile_repo, file=config)
+            print('git = "%s"' % repo, file=config)
+            print('rev = "%s"' % revision, file=config)
+            print('replace-with = "yocto-vendored-sources"', file=config)
+
+    # check if there are any git repos in the lock file that were not visited
+    # in the previous loop, when the source replacement was created, and warn about it
+    for lf_repo, found_in_src_uri in lockfile_git_repos.items():
+        if not found_in_src_uri:
+            bb.warn(f"{lf_repo} is present in lockfile, but not found in SRC_URI")
 
-    with open(lockfile, "w") as f:
-        f.writelines(newlines)
 }
+
+do_configure[cleandirs] += "${UNPACKDIR}/yocto-vendored-source-crates"
 do_configure[postfuncs] += "cargo_common_do_patch_paths"
 
 do_compile:prepend () {