| Message ID | 20251001163457.49114-1-daniel.dragomir@windriver.com |
|---|---|
| State | Awaiting Upstream |
| Delegated to: | Steve Sakoman |
| Headers | show |
| Series | [scarthgap] wic/engine: fix copying directories into wic image with ext* partition | expand |
Is this also an issue on master? If so, you will need to submit this patch for master before I can take it for scarthgap. If not, can you explain why it isn't needed there? Thanks, Steve On Wed, Oct 1, 2025 at 2:25 PM Dragomir, Daniel via lists.openembedded.org <daniel.dragomir=windriver.com@lists.openembedded.org> wrote: > > wic uses debugfs to write on ext* partitions, but debugfs can only > write to the current working directory and it cannot copy complete > directory trees. Running 'wic ls' on a copied directory show this: > -l: Ext2 inode is not a directory > > Fix this by creating a command list for debugfs (-f parameter) when > recursive parsing the host directory in order to create a similar > directory structure (mkdir) and copy files (write) on each level > into the destination directory from the wic's ext* partition. > > Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com> > --- > scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++--------- > 1 file changed, 49 insertions(+), 14 deletions(-) > > diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py > index b9e60cbe4e..9d596be3a7 100644 > --- a/scripts/lib/wic/engine.py > +++ b/scripts/lib/wic/engine.py > @@ -345,29 +345,64 @@ class Disk: > path)) > > def copy(self, src, dest): > - """Copy partition image into wic image.""" > - pnum = dest.part if isinstance(src, str) else src.part > + """Copy files or directories to/from the vfat or ext* partition.""" > + pnum = dest.part if isinstance(src, str) else src.part > + partimg = self._get_part_image(pnum) > > if self.partitions[pnum].fstype.startswith('ext'): > - if isinstance(src, str): > - cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ > - format(os.path.dirname(dest.path), src, os.path.basename(src), > - self.debugfs, self._get_part_image(pnum)) > - else: # copy from wic > - # run both dump and rdump to support both files and directory > + if isinstance(src, str): # host to image case > + if os.path.isdir(src): > + base = os.path.abspath(src) > + base_parent = os.path.dirname(base) > + cmds = [] > + made = set() > + > + for root, dirs, files in os.walk(base): > + for fname in files: > + host_file = os.path.join(root, fname) > + rel = os.path.relpath(host_file, base_parent) > + dest_file = os.path.join(dest.path, rel) > + dest_dir = os.path.dirname(dest_file) > + > + # create dir structure (mkdir -p) > + parts = dest_dir.strip('/').split('/') > + cur = '' > + for p in parts: > + cur = cur + '/' + p > + if cur not in made: > + cmds.append(f'mkdir "{cur}"') > + made.add(cur) > + > + cmds.append(f'write "{host_file}" "{dest_file}"') > + > + # write script to a temp file > + with tempfile.NamedTemporaryFile(mode='w', delete=False, > + prefix='wic-debugfs-') as tf: > + for line in cmds: > + tf.write(line + '\n') > + scriptname = tf.name > + > + cmd = f"{self.debugfs} -w -f {scriptname} {partimg}" > + > + else: # single file > + cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ > + format(os.path.dirname(dest.path), src, > + os.path.basename(src), self.debugfs, partimg) > + > + else: # image to host case > cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ > format(os.path.dirname(src.path), src.path, > - dest, src.path, dest, self.debugfs, > - self._get_part_image(pnum)) > + dest, src.path, dest, self.debugfs, partimg) > + > else: # fat > if isinstance(src, str): > cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, > - self._get_part_image(pnum), > - src, dest.path) > + partimg, > + src, dest.path) > else: > cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, > - self._get_part_image(pnum), > - src.path, dest) > + partimg, > + src.path, dest) > > exec_cmd(cmd, as_shell=True) > self._put_part_image(pnum) > -- > 2.39.5 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#224331): https://lists.openembedded.org/g/openembedded-core/message/224331 > Mute This Topic: https://lists.openembedded.org/mt/115542019/3620601 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [steve@sakoman.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Yes, this is an issue on master/master-next too. I tested and the same patch can be applied fine on both master and scarthgap branches. Regards, Daniel ________________________________ From: Steve Sakoman <steve@sakoman.com> Sent: Thursday, October 2, 2025 5:55 PM To: Dragomir, Daniel <Daniel.Dragomir@windriver.com> Cc: openembedded-core@lists.openembedded.org <openembedded-core@lists.openembedded.org> Subject: Re: [OE-core][scarthgap][PATCH] wic/engine: fix copying directories into wic image with ext* partition Is this also an issue on master? If so, you will need to submit this patch for master before I can take it for scarthgap. If not, can you explain why it isn't needed there? Thanks, Steve On Wed, Oct 1, 2025 at 2:25 PM Dragomir, Daniel via lists.openembedded.org <daniel.dragomir=windriver.com@lists.openembedded.org> wrote: > > wic uses debugfs to write on ext* partitions, but debugfs can only > write to the current working directory and it cannot copy complete > directory trees. Running 'wic ls' on a copied directory show this: > -l: Ext2 inode is not a directory > > Fix this by creating a command list for debugfs (-f parameter) when > recursive parsing the host directory in order to create a similar > directory structure (mkdir) and copy files (write) on each level > into the destination directory from the wic's ext* partition. > > Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com> > --- > scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++--------- > 1 file changed, 49 insertions(+), 14 deletions(-) > > diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py > index b9e60cbe4e..9d596be3a7 100644 > --- a/scripts/lib/wic/engine.py > +++ b/scripts/lib/wic/engine.py > @@ -345,29 +345,64 @@ class Disk: > path)) > > def copy(self, src, dest): > - """Copy partition image into wic image.""" > - pnum = dest.part if isinstance(src, str) else src.part > + """Copy files or directories to/from the vfat or ext* partition.""" > + pnum = dest.part if isinstance(src, str) else src.part > + partimg = self._get_part_image(pnum) > > if self.partitions[pnum].fstype.startswith('ext'): > - if isinstance(src, str): > - cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ > - format(os.path.dirname(dest.path), src, os.path.basename(src), > - self.debugfs, self._get_part_image(pnum)) > - else: # copy from wic > - # run both dump and rdump to support both files and directory > + if isinstance(src, str): # host to image case > + if os.path.isdir(src): > + base = os.path.abspath(src) > + base_parent = os.path.dirname(base) > + cmds = [] > + made = set() > + > + for root, dirs, files in os.walk(base): > + for fname in files: > + host_file = os.path.join(root, fname) > + rel = os.path.relpath(host_file, base_parent) > + dest_file = os.path.join(dest.path, rel) > + dest_dir = os.path.dirname(dest_file) > + > + # create dir structure (mkdir -p) > + parts = dest_dir.strip('/').split('/') > + cur = '' > + for p in parts: > + cur = cur + '/' + p > + if cur not in made: > + cmds.append(f'mkdir "{cur}"') > + made.add(cur) > + > + cmds.append(f'write "{host_file}" "{dest_file}"') > + > + # write script to a temp file > + with tempfile.NamedTemporaryFile(mode='w', delete=False, > + prefix='wic-debugfs-') as tf: > + for line in cmds: > + tf.write(line + '\n') > + scriptname = tf.name > + > + cmd = f"{self.debugfs} -w -f {scriptname} {partimg}" > + > + else: # single file > + cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ > + format(os.path.dirname(dest.path), src, > + os.path.basename(src), self.debugfs, partimg) > + > + else: # image to host case > cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ > format(os.path.dirname(src.path), src.path, > - dest, src.path, dest, self.debugfs, > - self._get_part_image(pnum)) > + dest, src.path, dest, self.debugfs, partimg) > + > else: # fat > if isinstance(src, str): > cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, > - self._get_part_image(pnum), > - src, dest.path) > + partimg, > + src, dest.path) > else: > cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, > - self._get_part_image(pnum), > - src.path, dest) > + partimg, > + src.path, dest) > > exec_cmd(cmd, as_shell=True) > self._put_part_image(pnum) > -- > 2.39.5 > > > >
On Fri, Oct 3, 2025 at 12:13 AM Dragomir, Daniel <Daniel.Dragomir@windriver.com> wrote: > > Yes, this is an issue on master/master-next too. > I tested and the same patch can be applied fine on both master and scarthgap branches. Thanks for checking. Please submit the patch for the master branch. It would be helpful if you would then ping me when you see that it has been accepted. I'll try to remember to watch for it, but a ping would ensure I don't miss it :-) Steve > > Regards, > Daniel > > ________________________________ > From: Steve Sakoman <steve@sakoman.com> > Sent: Thursday, October 2, 2025 5:55 PM > To: Dragomir, Daniel <Daniel.Dragomir@windriver.com> > Cc: openembedded-core@lists.openembedded.org <openembedded-core@lists.openembedded.org> > Subject: Re: [OE-core][scarthgap][PATCH] wic/engine: fix copying directories into wic image with ext* partition > > Is this also an issue on master? If so, you will need to submit this > patch for master before I can take it for scarthgap. If not, can you > explain why it isn't needed there? > > Thanks, > > Steve > > On Wed, Oct 1, 2025 at 2:25 PM Dragomir, Daniel via > lists.openembedded.org > <daniel.dragomir=windriver.com@lists.openembedded.org> wrote: > > > > wic uses debugfs to write on ext* partitions, but debugfs can only > > write to the current working directory and it cannot copy complete > > directory trees. Running 'wic ls' on a copied directory show this: > > -l: Ext2 inode is not a directory > > > > Fix this by creating a command list for debugfs (-f parameter) when > > recursive parsing the host directory in order to create a similar > > directory structure (mkdir) and copy files (write) on each level > > into the destination directory from the wic's ext* partition. > > > > Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com> > > --- > > scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++--------- > > 1 file changed, 49 insertions(+), 14 deletions(-) > > > > diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py > > index b9e60cbe4e..9d596be3a7 100644 > > --- a/scripts/lib/wic/engine.py > > +++ b/scripts/lib/wic/engine.py > > @@ -345,29 +345,64 @@ class Disk: > > path)) > > > > def copy(self, src, dest): > > - """Copy partition image into wic image.""" > > - pnum = dest.part if isinstance(src, str) else src.part > > + """Copy files or directories to/from the vfat or ext* partition.""" > > + pnum = dest.part if isinstance(src, str) else src.part > > + partimg = self._get_part_image(pnum) > > > > if self.partitions[pnum].fstype.startswith('ext'): > > - if isinstance(src, str): > > - cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ > > - format(os.path.dirname(dest.path), src, os.path.basename(src), > > - self.debugfs, self._get_part_image(pnum)) > > - else: # copy from wic > > - # run both dump and rdump to support both files and directory > > + if isinstance(src, str): # host to image case > > + if os.path.isdir(src): > > + base = os.path.abspath(src) > > + base_parent = os.path.dirname(base) > > + cmds = [] > > + made = set() > > + > > + for root, dirs, files in os.walk(base): > > + for fname in files: > > + host_file = os.path.join(root, fname) > > + rel = os.path.relpath(host_file, base_parent) > > + dest_file = os.path.join(dest.path, rel) > > + dest_dir = os.path.dirname(dest_file) > > + > > + # create dir structure (mkdir -p) > > + parts = dest_dir.strip('/').split('/') > > + cur = '' > > + for p in parts: > > + cur = cur + '/' + p > > + if cur not in made: > > + cmds.append(f'mkdir "{cur}"') > > + made.add(cur) > > + > > + cmds.append(f'write "{host_file}" "{dest_file}"') > > + > > + # write script to a temp file > > + with tempfile.NamedTemporaryFile(mode='w', delete=False, > > + prefix='wic-debugfs-') as tf: > > + for line in cmds: > > + tf.write(line + '\n') > > + scriptname = tf.name > > + > > + cmd = f"{self.debugfs} -w -f {scriptname} {partimg}" > > + > > + else: # single file > > + cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ > > + format(os.path.dirname(dest.path), src, > > + os.path.basename(src), self.debugfs, partimg) > > + > > + else: # image to host case > > cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ > > format(os.path.dirname(src.path), src.path, > > - dest, src.path, dest, self.debugfs, > > - self._get_part_image(pnum)) > > + dest, src.path, dest, self.debugfs, partimg) > > + > > else: # fat > > if isinstance(src, str): > > cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, > > - self._get_part_image(pnum), > > - src, dest.path) > > + partimg, > > + src, dest.path) > > else: > > cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, > > - self._get_part_image(pnum), > > - src.path, dest) > > + partimg, > > + src.path, dest) > > > > exec_cmd(cmd, as_shell=True) > > self._put_part_image(pnum) > > -- > > 2.39.5 > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#224331): https://lists.openembedded.org/g/openembedded-core/message/224331 > > Mute This Topic: https://lists.openembedded.org/mt/115542019/3620601 > > Group Owner: openembedded-core+owner@lists.openembedded.org > > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [steve@sakoman.com] > > -=-=-=-=-=-=-=-=-=-=-=- > >
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py index b9e60cbe4e..9d596be3a7 100644 --- a/scripts/lib/wic/engine.py +++ b/scripts/lib/wic/engine.py @@ -345,29 +345,64 @@ class Disk: path)) def copy(self, src, dest): - """Copy partition image into wic image.""" - pnum = dest.part if isinstance(src, str) else src.part + """Copy files or directories to/from the vfat or ext* partition.""" + pnum = dest.part if isinstance(src, str) else src.part + partimg = self._get_part_image(pnum) if self.partitions[pnum].fstype.startswith('ext'): - if isinstance(src, str): - cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ - format(os.path.dirname(dest.path), src, os.path.basename(src), - self.debugfs, self._get_part_image(pnum)) - else: # copy from wic - # run both dump and rdump to support both files and directory + if isinstance(src, str): # host to image case + if os.path.isdir(src): + base = os.path.abspath(src) + base_parent = os.path.dirname(base) + cmds = [] + made = set() + + for root, dirs, files in os.walk(base): + for fname in files: + host_file = os.path.join(root, fname) + rel = os.path.relpath(host_file, base_parent) + dest_file = os.path.join(dest.path, rel) + dest_dir = os.path.dirname(dest_file) + + # create dir structure (mkdir -p) + parts = dest_dir.strip('/').split('/') + cur = '' + for p in parts: + cur = cur + '/' + p + if cur not in made: + cmds.append(f'mkdir "{cur}"') + made.add(cur) + + cmds.append(f'write "{host_file}" "{dest_file}"') + + # write script to a temp file + with tempfile.NamedTemporaryFile(mode='w', delete=False, + prefix='wic-debugfs-') as tf: + for line in cmds: + tf.write(line + '\n') + scriptname = tf.name + + cmd = f"{self.debugfs} -w -f {scriptname} {partimg}" + + else: # single file + cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ + format(os.path.dirname(dest.path), src, + os.path.basename(src), self.debugfs, partimg) + + else: # image to host case cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ format(os.path.dirname(src.path), src.path, - dest, src.path, dest, self.debugfs, - self._get_part_image(pnum)) + dest, src.path, dest, self.debugfs, partimg) + else: # fat if isinstance(src, str): cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, - self._get_part_image(pnum), - src, dest.path) + partimg, + src, dest.path) else: cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, - self._get_part_image(pnum), - src.path, dest) + partimg, + src.path, dest) exec_cmd(cmd, as_shell=True) self._put_part_image(pnum)
wic uses debugfs to write on ext* partitions, but debugfs can only write to the current working directory and it cannot copy complete directory trees. Running 'wic ls' on a copied directory show this: -l: Ext2 inode is not a directory Fix this by creating a command list for debugfs (-f parameter) when recursive parsing the host directory in order to create a similar directory structure (mkdir) and copy files (write) on each level into the destination directory from the wic's ext* partition. Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com> --- scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 14 deletions(-)