diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py index fb6278439..d74c20df4 100644 --- a/lib/bb/fetch/__init__.py +++ b/lib/bb/fetch/__init__.py @@ -1413,6 +1413,15 @@ class FetchData(object): self.donestamp = basepath + '.done' self.lockfile = basepath + '.lock' + def checksum_expected(self): + """ + Is any checksum given for this url? + """ + for checksum_id in CHECKSUM_LIST: + if getattr(self, "%s_expected" % checksum_id): + return True + return False + def setup_revisions(self, d): self.revision = srcrev_internal_helper(self, d, self.name) diff --git a/lib/bb/fetch/local.py b/lib/bb/fetch/local.py index dfc5b564c..ca9f91497 100644 --- a/lib/bb/fetch/local.py +++ b/lib/bb/fetch/local.py @@ -14,6 +14,8 @@ BitBake build tools. # import os +import hashlib +import json import urllib.request, urllib.parse, urllib.error import bb import bb.utils @@ -36,6 +38,82 @@ class Local(FetchMethod): raise bb.fetch.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url) return + def checksum_stamp(self, ud, d): + """ + Return the stamp file caching the checksums of ud.localpath, or None. + """ + dldir = d.getVar("DL_DIR") + if not dldir or not ud.localpath: + return None + path = os.path.abspath(ud.localpath) + return os.path.join(dldir, "local-checksums", + hashlib.sha256(path.encode("utf-8")).hexdigest()) + + def checksum_filestat(self, st): + """ + Return a list of the file's stat information that is used to detect changes. + """ + return [st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns] + + def read_checksum_stamp(self, ud, d, st): + """ + Return the checksums cached in the stamp file for ud.localpath, or {} + if there are none or the file has changed since they were computed. + """ + stamp = self.checksum_stamp(ud, d) + if not stamp: + return {} + try: + with open(stamp) as f: + cached = json.load(f) + if cached["stat"] == self.checksum_filestat(st) and isinstance(cached["checksums"], dict): + return cached["checksums"] + except (OSError, ValueError, KeyError, TypeError): + pass + return {} + + def write_checksum_stamp(self, ud, d, st, checksums): + """ + Cache the checksums computed for ud.localpath, which had the stat + result st before it was hashed, in the stamp file. + """ + stamp = self.checksum_stamp(ud, d) + if not stamp: + return + + # Written via rename as no lock is held for file:// urls. Errors + # aren't fatal, the file is just hashed again next time. + tmpstamp = "%s.%d" % (stamp, os.getpid()) + try: + bb.utils.mkdirhier(os.path.dirname(stamp)) + with open(tmpstamp, "w") as f: + json.dump({"path": ud.localpath, "stat": self.checksum_filestat(st), "checksums": checksums}, f) + os.rename(tmpstamp, stamp) + except OSError: + try: + os.unlink(tmpstamp) + except OSError: + pass + + def verify_donestamp(self, ud, d): + # file:// urls have no donestamp; verify any checksum given in the url + # against the file itself. A missing file is reported by download() + # with the searched paths, so there's nothing to check here. A mismatch + # raises ChecksumError rather than returning False, as there's nothing + # to re-download. + if not ud.checksum_expected(): + return True + try: + st = os.stat(ud.localpath) + except OSError: + return True + + precomputed = self.read_checksum_stamp(ud, d, st) + checksums = bb.fetch.verify_checksum(ud, d, precomputed) + if checksums and checksums != precomputed: + self.write_checksum_stamp(ud, d, st, checksums) + return True + def localpath(self, urldata, d): """ Return the local filename of a given url assuming a successful fetch. @@ -87,5 +165,9 @@ class Local(FetchMethod): return False def clean(self, urldata, d): + # The file is the user's; only remove our cached checksums for it + stamp = self.checksum_stamp(urldata, d) + if stamp: + bb.utils.remove(stamp) return diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 71b0a63fe..cbb47849c 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -21,6 +21,7 @@ import subprocess import json import tarfile import threading +import time from bb.fetch import URI import bb import bb.utils @@ -876,6 +877,114 @@ class FetcherLocalTest(FetcherTest): with self.subTest(striplevel=repr(value)): self.assertInvalidStriplevel(value) + def writeSumfile(self, content, dirname=None): + path = os.path.join(dirname or self.localsrcdir, 'sumfile') + with open(path, 'wb') as f: + f.write(content) + return path, hashlib.sha256(content).hexdigest() + + def localChecksumStamps(self): + stampdir = os.path.join(self.dldir, 'local-checksums') + return os.listdir(stampdir) if os.path.exists(stampdir) else [] + + def test_local_checksum_match(self): + srcpath, good = self.writeSumfile(b"file:// checksum match test\n") + fetcher = bb.fetch.Fetch(['file://sumfile;sha256sum=' + good], self.d) + fetcher.download() + # Only the checksum stamp should be created in DL_DIR + self.assertEqual(os.listdir(self.dldir), ['local-checksums']) + stamps = self.localChecksumStamps() + self.assertEqual(len(stamps), 1) + # The stamp must follow the umask, so a shared DL_DIR can re-use it + umask = os.umask(0) + os.umask(umask) + mode = os.stat(os.path.join(self.dldir, 'local-checksums', stamps[0])).st_mode + self.assertEqual(mode & 0o777, 0o666 & ~umask) + # Cleaning removes the stamp but not the user's file + fetcher.clean() + self.assertEqual(self.localChecksumStamps(), []) + self.assertTrue(os.path.exists(srcpath)) + + def test_local_checksum_no_checksum_no_stamp(self): + self.writeSumfile(b"file:// no checksum test\n") + bb.fetch.Fetch(['file://sumfile'], self.d).download() + self.assertEqual(os.listdir(self.dldir), []) + + def test_local_checksum_stamp_write_failure(self): + # A failure to write the stamp isn't fatal and leaves nothing behind + srcpath, good = self.writeSumfile(b"file:// checksum stamp failure test\n") + with unittest.mock.patch('os.rename', side_effect=OSError("rename failed")): + bb.fetch.Fetch(['file://sumfile;sha256sum=' + good], self.d).download() + self.assertEqual(self.localChecksumStamps(), []) + + def test_local_checksum_unreachable_file(self): + # A path below a regular file can't be stat()ed; this should be + # reported as a missing file, not as an OSError + srcpath, good = self.writeSumfile(b"file:// checksum unreachable test\n") + fetcher = bb.fetch.Fetch(['file://%s/sub;sha256sum=%s' % (srcpath, good)], self.d) + with self.assertRaises(bb.fetch.FetchError): + fetcher.download() + + def test_local_checksum_unchanged_not_rehashed(self): + stampdir, good = self.writeSumfile(b"file:// checksum rehash test\n") + url = 'file://sumfile;sha256sum=' + good + with unittest.mock.patch('bb.utils.sha256_file', + wraps=bb.utils.sha256_file) as sha256_file: + bb.fetch.Fetch([url], self.d).download() + self.assertEqual(sha256_file.call_count, 1) + # A later fetch re-uses the checksums stored in the stamp + bb.fetch.Fetch([url], self.d).download() + self.assertEqual(sha256_file.call_count, 1) + # A wrong checksum in the url is still caught without re-hashing + with self.assertRaises(bb.fetch.ChecksumError): + bb.fetch.Fetch(['file://sumfile;sha256sum=' + "0" * 64], self.d).download() + self.assertEqual(sha256_file.call_count, 1) + + def test_local_checksum_modified_keeping_mtime(self): + srcpath, good = self.writeSumfile(b"file:// checksum original content\n") + url = 'file://sumfile;sha256sum=' + good + bb.fetch.Fetch([url], self.d).download() + + # Replace the content in place with the same size and restore the + # exact original mtime, as rsync -a or touch -r could; only ctime + # changes. Filesystem timestamps may only advance once per timer + # tick, so wait for longer than that first. + st = os.stat(srcpath) + time.sleep(0.05) + with open(srcpath, 'wb') as f: + f.write(b"file:// checksum modified content\n") + os.utime(srcpath, ns=(st.st_atime_ns, st.st_mtime_ns)) + with self.assertRaises(bb.fetch.ChecksumError): + bb.fetch.Fetch([url], self.d).download() + self.assertTrue(os.path.exists(srcpath)) + + def test_local_checksum_mismatch(self): + content = b"file:// checksum mismatch test\n" + srcpath, _ = self.writeSumfile(content) + md5 = hashlib.md5(content).hexdigest() + # A correct checksum must not hide a wrong one + url = 'file://sumfile;md5sum=%s;sha256sum=%s' % (md5, "0" * 64) + fetcher = bb.fetch.Fetch([url], self.d) + with self.assertRaises(bb.fetch.ChecksumError) as cm: + fetcher.download() + self.assertIn("has sha256 checksum", str(cm.exception)) + # The user's file must not be renamed to *_bad-checksum_* + self.assertTrue(os.path.exists(srcpath)) + self.assertEqual(self.localChecksumStamps(), []) + + def test_local_checksum_same_name_different_dirs(self): + # Files with the same name must not share a stamp + sums = {} + for name in ('a', 'b'): + dirname = os.path.join(self.localsrcdir, 'sumdir-' + name) + os.mkdir(dirname) + sums[name] = self.writeSumfile(b"file:// checksum dir %s\n" % name.encode(), dirname) + for srcpath, good in sums.values(): + bb.fetch.Fetch(['file://%s;sha256sum=%s' % (srcpath, good)], self.d).download() + self.assertEqual(len(self.localChecksumStamps()), 2) + with self.assertRaises(bb.fetch.ChecksumError): + bb.fetch.Fetch(['file://%s;sha256sum=%s' % (sums['a'][0], sums['b'][1])], self.d).download() + def dummyGitTest(self, suffix): # Create dummy local Git repo src_dir = tempfile.mkdtemp(dir=self.tempdir,