diff mbox series

[scarthgap] wic/engine: fix copying directories into wic image with ext* partition

Message ID 20251001163457.49114-1-daniel.dragomir@windriver.com
State Awaiting Upstream
Delegated to: Steve Sakoman
Headers show
Series [scarthgap] wic/engine: fix copying directories into wic image with ext* partition | expand

Commit Message

Dragomir, Daniel Oct. 1, 2025, 4:34 p.m. UTC
wic uses debugfs to write on ext* partitions, but debugfs can only
write to the current working directory and it cannot copy complete
directory trees. Running 'wic ls' on a copied directory show this:
    -l: Ext2 inode is not a directory

Fix this by creating a command list for debugfs (-f parameter) when
recursive parsing the host directory in order to create a similar
directory structure (mkdir) and copy files (write) on each level
into the destination directory from the wic's ext* partition.

Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com>
---
 scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++---------
 1 file changed, 49 insertions(+), 14 deletions(-)

Comments

Steve Sakoman Oct. 2, 2025, 2:55 p.m. UTC | #1
Is this also an issue on master?  If so, you will need to submit this
patch for master before I can take it for scarthgap.  If not, can you
explain why it isn't needed there?

Thanks,

Steve

On Wed, Oct 1, 2025 at 2:25 PM Dragomir, Daniel via
lists.openembedded.org
<daniel.dragomir=windriver.com@lists.openembedded.org> wrote:
>
> wic uses debugfs to write on ext* partitions, but debugfs can only
> write to the current working directory and it cannot copy complete
> directory trees. Running 'wic ls' on a copied directory show this:
>     -l: Ext2 inode is not a directory
>
> Fix this by creating a command list for debugfs (-f parameter) when
> recursive parsing the host directory in order to create a similar
> directory structure (mkdir) and copy files (write) on each level
> into the destination directory from the wic's ext* partition.
>
> Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com>
> ---
>  scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++---------
>  1 file changed, 49 insertions(+), 14 deletions(-)
>
> diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
> index b9e60cbe4e..9d596be3a7 100644
> --- a/scripts/lib/wic/engine.py
> +++ b/scripts/lib/wic/engine.py
> @@ -345,29 +345,64 @@ class Disk:
>                                                     path))
>
>      def copy(self, src, dest):
> -        """Copy partition image into wic image."""
> -        pnum =  dest.part if isinstance(src, str) else src.part
> +        """Copy files or directories to/from the vfat or ext* partition."""
> +        pnum = dest.part if isinstance(src, str) else src.part
> +        partimg = self._get_part_image(pnum)
>
>          if self.partitions[pnum].fstype.startswith('ext'):
> -            if isinstance(src, str):
> -                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
> -                      format(os.path.dirname(dest.path), src, os.path.basename(src),
> -                             self.debugfs, self._get_part_image(pnum))
> -            else: # copy from wic
> -                # run both dump and rdump to support both files and directory
> +            if isinstance(src, str): # host to image case
> +                if os.path.isdir(src):
> +                    base = os.path.abspath(src)
> +                    base_parent = os.path.dirname(base)
> +                    cmds = []
> +                    made = set()
> +
> +                    for root, dirs, files in os.walk(base):
> +                        for fname in files:
> +                            host_file = os.path.join(root, fname)
> +                            rel = os.path.relpath(host_file, base_parent)
> +                            dest_file = os.path.join(dest.path, rel)
> +                            dest_dir = os.path.dirname(dest_file)
> +
> +                            # create dir structure (mkdir -p)
> +                            parts = dest_dir.strip('/').split('/')
> +                            cur = ''
> +                            for p in parts:
> +                                cur = cur + '/' + p
> +                                if cur not in made:
> +                                    cmds.append(f'mkdir "{cur}"')
> +                                    made.add(cur)
> +
> +                            cmds.append(f'write "{host_file}" "{dest_file}"')
> +
> +                    # write script to a temp file
> +                    with tempfile.NamedTemporaryFile(mode='w', delete=False,
> +                                                     prefix='wic-debugfs-') as tf:
> +                        for line in cmds:
> +                            tf.write(line + '\n')
> +                        scriptname = tf.name
> +
> +                    cmd = f"{self.debugfs} -w -f {scriptname} {partimg}"
> +
> +                else: # single file
> +                    cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
> +                          format(os.path.dirname(dest.path), src,
> +                                 os.path.basename(src), self.debugfs, partimg)
> +
> +            else: # image to host case
>                  cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
>                        format(os.path.dirname(src.path), src.path,
> -                             dest, src.path, dest, self.debugfs,
> -                             self._get_part_image(pnum))
> +                             dest, src.path, dest, self.debugfs, partimg)
> +
>          else: # fat
>              if isinstance(src, str):
>                  cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
> -                                                  self._get_part_image(pnum),
> -                                                  src, dest.path)
> +                                                      partimg,
> +                                                      src, dest.path)
>              else:
>                  cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
> -                                                  self._get_part_image(pnum),
> -                                                  src.path, dest)
> +                                                      partimg,
> +                                                      src.path, dest)
>
>          exec_cmd(cmd, as_shell=True)
>          self._put_part_image(pnum)
> --
> 2.39.5
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#224331): https://lists.openembedded.org/g/openembedded-core/message/224331
> Mute This Topic: https://lists.openembedded.org/mt/115542019/3620601
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [steve@sakoman.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
Dragomir, Daniel Oct. 3, 2025, 7:13 a.m. UTC | #2
Yes, this is an issue on master/master-next too.
I tested and the same patch can be applied fine on both master and scarthgap branches.

Regards,
Daniel

________________________________
From: Steve Sakoman <steve@sakoman.com>
Sent: Thursday, October 2, 2025 5:55 PM
To: Dragomir, Daniel <Daniel.Dragomir@windriver.com>
Cc: openembedded-core@lists.openembedded.org <openembedded-core@lists.openembedded.org>
Subject: Re: [OE-core][scarthgap][PATCH] wic/engine: fix copying directories into wic image with ext* partition

Is this also an issue on master?  If so, you will need to submit this
patch for master before I can take it for scarthgap.  If not, can you
explain why it isn't needed there?

Thanks,

Steve

On Wed, Oct 1, 2025 at 2:25 PM Dragomir, Daniel via
lists.openembedded.org
<daniel.dragomir=windriver.com@lists.openembedded.org> wrote:
>
> wic uses debugfs to write on ext* partitions, but debugfs can only
> write to the current working directory and it cannot copy complete
> directory trees. Running 'wic ls' on a copied directory show this:
>     -l: Ext2 inode is not a directory
>
> Fix this by creating a command list for debugfs (-f parameter) when
> recursive parsing the host directory in order to create a similar
> directory structure (mkdir) and copy files (write) on each level
> into the destination directory from the wic's ext* partition.
>
> Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com>
> ---
>  scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++---------
>  1 file changed, 49 insertions(+), 14 deletions(-)
>
> diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
> index b9e60cbe4e..9d596be3a7 100644
> --- a/scripts/lib/wic/engine.py
> +++ b/scripts/lib/wic/engine.py
> @@ -345,29 +345,64 @@ class Disk:
>                                                     path))
>
>      def copy(self, src, dest):
> -        """Copy partition image into wic image."""
> -        pnum =  dest.part if isinstance(src, str) else src.part
> +        """Copy files or directories to/from the vfat or ext* partition."""
> +        pnum = dest.part if isinstance(src, str) else src.part
> +        partimg = self._get_part_image(pnum)
>
>          if self.partitions[pnum].fstype.startswith('ext'):
> -            if isinstance(src, str):
> -                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
> -                      format(os.path.dirname(dest.path), src, os.path.basename(src),
> -                             self.debugfs, self._get_part_image(pnum))
> -            else: # copy from wic
> -                # run both dump and rdump to support both files and directory
> +            if isinstance(src, str): # host to image case
> +                if os.path.isdir(src):
> +                    base = os.path.abspath(src)
> +                    base_parent = os.path.dirname(base)
> +                    cmds = []
> +                    made = set()
> +
> +                    for root, dirs, files in os.walk(base):
> +                        for fname in files:
> +                            host_file = os.path.join(root, fname)
> +                            rel = os.path.relpath(host_file, base_parent)
> +                            dest_file = os.path.join(dest.path, rel)
> +                            dest_dir = os.path.dirname(dest_file)
> +
> +                            # create dir structure (mkdir -p)
> +                            parts = dest_dir.strip('/').split('/')
> +                            cur = ''
> +                            for p in parts:
> +                                cur = cur + '/' + p
> +                                if cur not in made:
> +                                    cmds.append(f'mkdir "{cur}"')
> +                                    made.add(cur)
> +
> +                            cmds.append(f'write "{host_file}" "{dest_file}"')
> +
> +                    # write script to a temp file
> +                    with tempfile.NamedTemporaryFile(mode='w', delete=False,
> +                                                     prefix='wic-debugfs-') as tf:
> +                        for line in cmds:
> +                            tf.write(line + '\n')
> +                        scriptname = tf.name
> +
> +                    cmd = f"{self.debugfs} -w -f {scriptname} {partimg}"
> +
> +                else: # single file
> +                    cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
> +                          format(os.path.dirname(dest.path), src,
> +                                 os.path.basename(src), self.debugfs, partimg)
> +
> +            else: # image to host case
>                  cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
>                        format(os.path.dirname(src.path), src.path,
> -                             dest, src.path, dest, self.debugfs,
> -                             self._get_part_image(pnum))
> +                             dest, src.path, dest, self.debugfs, partimg)
> +
>          else: # fat
>              if isinstance(src, str):
>                  cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
> -                                                  self._get_part_image(pnum),
> -                                                  src, dest.path)
> +                                                      partimg,
> +                                                      src, dest.path)
>              else:
>                  cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
> -                                                  self._get_part_image(pnum),
> -                                                  src.path, dest)
> +                                                      partimg,
> +                                                      src.path, dest)
>
>          exec_cmd(cmd, as_shell=True)
>          self._put_part_image(pnum)
> --
> 2.39.5
>
>
> 
>
Steve Sakoman Oct. 3, 2025, 4:43 p.m. UTC | #3
On Fri, Oct 3, 2025 at 12:13 AM Dragomir, Daniel
<Daniel.Dragomir@windriver.com> wrote:
>
> Yes, this is an issue on master/master-next too.
> I tested and the same patch can be applied fine on both master and scarthgap branches.

Thanks for checking.

Please submit the patch for the master branch.  It would be helpful if
you would then ping me when you see that it has been accepted.  I'll
try to remember to watch for it, but a ping would ensure I don't miss
it :-)

Steve

>
> Regards,
> Daniel
>
> ________________________________
> From: Steve Sakoman <steve@sakoman.com>
> Sent: Thursday, October 2, 2025 5:55 PM
> To: Dragomir, Daniel <Daniel.Dragomir@windriver.com>
> Cc: openembedded-core@lists.openembedded.org <openembedded-core@lists.openembedded.org>
> Subject: Re: [OE-core][scarthgap][PATCH] wic/engine: fix copying directories into wic image with ext* partition
>
> Is this also an issue on master?  If so, you will need to submit this
> patch for master before I can take it for scarthgap.  If not, can you
> explain why it isn't needed there?
>
> Thanks,
>
> Steve
>
> On Wed, Oct 1, 2025 at 2:25 PM Dragomir, Daniel via
> lists.openembedded.org
> <daniel.dragomir=windriver.com@lists.openembedded.org> wrote:
> >
> > wic uses debugfs to write on ext* partitions, but debugfs can only
> > write to the current working directory and it cannot copy complete
> > directory trees. Running 'wic ls' on a copied directory show this:
> >     -l: Ext2 inode is not a directory
> >
> > Fix this by creating a command list for debugfs (-f parameter) when
> > recursive parsing the host directory in order to create a similar
> > directory structure (mkdir) and copy files (write) on each level
> > into the destination directory from the wic's ext* partition.
> >
> > Signed-off-by: Daniel Dragomir <daniel.dragomir@windriver.com>
> > ---
> >  scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++---------
> >  1 file changed, 49 insertions(+), 14 deletions(-)
> >
> > diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
> > index b9e60cbe4e..9d596be3a7 100644
> > --- a/scripts/lib/wic/engine.py
> > +++ b/scripts/lib/wic/engine.py
> > @@ -345,29 +345,64 @@ class Disk:
> >                                                     path))
> >
> >      def copy(self, src, dest):
> > -        """Copy partition image into wic image."""
> > -        pnum =  dest.part if isinstance(src, str) else src.part
> > +        """Copy files or directories to/from the vfat or ext* partition."""
> > +        pnum = dest.part if isinstance(src, str) else src.part
> > +        partimg = self._get_part_image(pnum)
> >
> >          if self.partitions[pnum].fstype.startswith('ext'):
> > -            if isinstance(src, str):
> > -                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
> > -                      format(os.path.dirname(dest.path), src, os.path.basename(src),
> > -                             self.debugfs, self._get_part_image(pnum))
> > -            else: # copy from wic
> > -                # run both dump and rdump to support both files and directory
> > +            if isinstance(src, str): # host to image case
> > +                if os.path.isdir(src):
> > +                    base = os.path.abspath(src)
> > +                    base_parent = os.path.dirname(base)
> > +                    cmds = []
> > +                    made = set()
> > +
> > +                    for root, dirs, files in os.walk(base):
> > +                        for fname in files:
> > +                            host_file = os.path.join(root, fname)
> > +                            rel = os.path.relpath(host_file, base_parent)
> > +                            dest_file = os.path.join(dest.path, rel)
> > +                            dest_dir = os.path.dirname(dest_file)
> > +
> > +                            # create dir structure (mkdir -p)
> > +                            parts = dest_dir.strip('/').split('/')
> > +                            cur = ''
> > +                            for p in parts:
> > +                                cur = cur + '/' + p
> > +                                if cur not in made:
> > +                                    cmds.append(f'mkdir "{cur}"')
> > +                                    made.add(cur)
> > +
> > +                            cmds.append(f'write "{host_file}" "{dest_file}"')
> > +
> > +                    # write script to a temp file
> > +                    with tempfile.NamedTemporaryFile(mode='w', delete=False,
> > +                                                     prefix='wic-debugfs-') as tf:
> > +                        for line in cmds:
> > +                            tf.write(line + '\n')
> > +                        scriptname = tf.name
> > +
> > +                    cmd = f"{self.debugfs} -w -f {scriptname} {partimg}"
> > +
> > +                else: # single file
> > +                    cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
> > +                          format(os.path.dirname(dest.path), src,
> > +                                 os.path.basename(src), self.debugfs, partimg)
> > +
> > +            else: # image to host case
> >                  cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
> >                        format(os.path.dirname(src.path), src.path,
> > -                             dest, src.path, dest, self.debugfs,
> > -                             self._get_part_image(pnum))
> > +                             dest, src.path, dest, self.debugfs, partimg)
> > +
> >          else: # fat
> >              if isinstance(src, str):
> >                  cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
> > -                                                  self._get_part_image(pnum),
> > -                                                  src, dest.path)
> > +                                                      partimg,
> > +                                                      src, dest.path)
> >              else:
> >                  cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
> > -                                                  self._get_part_image(pnum),
> > -                                                  src.path, dest)
> > +                                                      partimg,
> > +                                                      src.path, dest)
> >
> >          exec_cmd(cmd, as_shell=True)
> >          self._put_part_image(pnum)
> > --
> > 2.39.5
> >
> >
> > -=-=-=-=-=-=-=-=-=-=-=-
> > Links: You receive all messages sent to this group.
> > View/Reply Online (#224331): https://lists.openembedded.org/g/openembedded-core/message/224331
> > Mute This Topic: https://lists.openembedded.org/mt/115542019/3620601
> > Group Owner: openembedded-core+owner@lists.openembedded.org
> > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [steve@sakoman.com]
> > -=-=-=-=-=-=-=-=-=-=-=-
> >
diff mbox series

Patch

diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index b9e60cbe4e..9d596be3a7 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -345,29 +345,64 @@  class Disk:
                                                    path))
 
     def copy(self, src, dest):
-        """Copy partition image into wic image."""
-        pnum =  dest.part if isinstance(src, str) else src.part
+        """Copy files or directories to/from the vfat or ext* partition."""
+        pnum = dest.part if isinstance(src, str) else src.part
+        partimg = self._get_part_image(pnum)
 
         if self.partitions[pnum].fstype.startswith('ext'):
-            if isinstance(src, str):
-                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
-                      format(os.path.dirname(dest.path), src, os.path.basename(src),
-                             self.debugfs, self._get_part_image(pnum))
-            else: # copy from wic
-                # run both dump and rdump to support both files and directory
+            if isinstance(src, str): # host to image case
+                if os.path.isdir(src):
+                    base = os.path.abspath(src)
+                    base_parent = os.path.dirname(base)
+                    cmds = []
+                    made = set()
+
+                    for root, dirs, files in os.walk(base):
+                        for fname in files:
+                            host_file = os.path.join(root, fname)
+                            rel = os.path.relpath(host_file, base_parent)
+                            dest_file = os.path.join(dest.path, rel)
+                            dest_dir = os.path.dirname(dest_file)
+
+                            # create dir structure (mkdir -p)
+                            parts = dest_dir.strip('/').split('/')
+                            cur = ''
+                            for p in parts:
+                                cur = cur + '/' + p
+                                if cur not in made:
+                                    cmds.append(f'mkdir "{cur}"')
+                                    made.add(cur)
+
+                            cmds.append(f'write "{host_file}" "{dest_file}"')
+
+                    # write script to a temp file
+                    with tempfile.NamedTemporaryFile(mode='w', delete=False,
+                                                     prefix='wic-debugfs-') as tf:
+                        for line in cmds:
+                            tf.write(line + '\n')
+                        scriptname = tf.name
+
+                    cmd = f"{self.debugfs} -w -f {scriptname} {partimg}"
+
+                else: # single file
+                    cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
+                          format(os.path.dirname(dest.path), src,
+                                 os.path.basename(src), self.debugfs, partimg)
+
+            else: # image to host case
                 cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
                       format(os.path.dirname(src.path), src.path,
-                             dest, src.path, dest, self.debugfs,
-                             self._get_part_image(pnum))
+                             dest, src.path, dest, self.debugfs, partimg)
+
         else: # fat
             if isinstance(src, str):
                 cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
-                                                  self._get_part_image(pnum),
-                                                  src, dest.path)
+                                                      partimg,
+                                                      src, dest.path)
             else:
                 cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
-                                                  self._get_part_image(pnum),
-                                                  src.path, dest)
+                                                      partimg,
+                                                      src.path, dest)
 
         exec_cmd(cmd, as_shell=True)
         self._put_part_image(pnum)