new file mode 100644
@@ -0,0 +1,183 @@
+Upstream-Status: Inappropriate [works around issues with rust and jemalloc]
+
+We see two deadlocks when running rust do_install due to jemalloc and
+pseudo conflicting at init time. We hack to hack the readlink() and
+syscall() intercepts to break the deadlock.
+
+#0 0x00007f3806c84bcf in __lll_lock_wait () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-uninative/x86_64-linux/lib/libc.so.6
+#1 0x00007f3806c8aec2 in pthread_mutex_lock () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-uninative/x86_64-linux/lib/libc.so.6
+#2 0x00005629da227a0e in malloc_mutex_lock_final (mutex=0x5629da45a288 <init_lock>) at include/jemalloc/internal/mutex.h:155
+#3 _rjem_je_malloc_mutex_lock_slow (mutex=0x5629da45a288 <init_lock>) at src/mutex.c:85
+#4 0x00005629da1ffddb in malloc_mutex_lock (tsdn=0x0, mutex=<optimized out>) at include/jemalloc/internal/mutex.h:221
+#5 malloc_init_hard () at src/jemalloc.c:1739
+#6 0x00005629da1fa2ee in malloc_init () at src/jemalloc.c:223
+#7 imalloc_init_check (sopts=<optimized out>, dopts=<optimized out>) at src/jemalloc.c:2229
+#8 imalloc (sopts=<optimized out>, dopts=<optimized out>) at src/jemalloc.c:2260
+#9 _rjem_je_malloc_default (size=101) at src/jemalloc.c:2289
+#10 0x00007f3806c998fe in strdup () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-uninative/x86_64-linux/lib/libc.so.6
+#11 0x00007f380b19da1b in pseudo_set_value () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#12 0x00007f380b19dbb9 in pseudo_init_util () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#13 0x00007f380b16bd01 in ?? () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#14 0x00007f380b19125d in readlink () from /home/pokybuild/yocto-worker/qemumips64/build/build/tmp/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#15 0x00005629da1fe07b in obtain_malloc_conf (which_source=2, buf=0x7ffca80e7cc0 "\320~\016\250\374\177") at src/jemalloc.c:980
+#16 malloc_conf_init_helper (sc_data=sc_data@entry=0x0, bin_shard_sizes=bin_shard_sizes@entry=0x0, initial_call=true, opts_cache=opts_cache@entry=0x7ffca80e6260, buf=<optimized out>,
+ buf@entry=0x7ffca80e7cc0 "\320~\016\250\374\177") at src/jemalloc.c:1038
+#17 0x00005629da1fdcac in malloc_conf_init (sc_data=0x7ffca80e6310, bin_shard_sizes=0x7ffca80e6280) at src/jemalloc.c:1445
+#18 malloc_init_hard_a0_locked () at src/jemalloc.c:1505
+#19 0x00005629da1fff46 in malloc_init_hard () at src/jemalloc.c:1750
+#20 0x00005629da1fa2ee in malloc_init () at src/jemalloc.c:223
+
+
+0x00007f36e6884bcf in __lll_lock_wait () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/libc.so.6
+(gdb) bt
+#0 0x00007f36e6884bcf in __lll_lock_wait () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/libc.so.6
+#1 0x00007f36e688aec2 in pthread_mutex_lock () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/libc.so.6
+#2 0x000055a537834a0e in malloc_mutex_lock_final (mutex=0x55a537a67288 <init_lock>) at include/jemalloc/internal/mutex.h:155
+#3 _rjem_je_malloc_mutex_lock_slow (mutex=0x55a537a67288 <init_lock>) at src/mutex.c:85
+#4 0x000055a53780cddb in malloc_mutex_lock (tsdn=0x0, mutex=<optimised out>) at include/jemalloc/internal/mutex.h:221
+#5 malloc_init_hard () at src/jemalloc.c:1739
+#6 0x000055a5378072ee in malloc_init () at src/jemalloc.c:223
+#7 imalloc_init_check (sopts=<optimised out>, dopts=<optimised out>) at src/jemalloc.c:2229
+#8 imalloc (sopts=<optimised out>, dopts=<optimised out>) at src/jemalloc.c:2260
+#9 _rjem_je_malloc_default (size=79) at src/jemalloc.c:2289
+#10 0x00007f36e68998fe in strdup () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/libc.so.6
+#11 0x00007f36eabf842e in pseudo_set_value () from /media/build1/poky/build/tmp-musl/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#12 0x00007f36eabf85e0 in pseudo_init_util () from /media/build1/poky/build/tmp-musl/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#13 0x00007f36eabc2645 in ?? () from /media/build1/poky/build/tmp-musl/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#14 0x00007f36eabd1055 in syscall () from /media/build1/poky/build/tmp-musl/sysroots-components/x86_64/pseudo-native/usr/lib/pseudo/lib64/libpseudo.so
+#15 0x000055a5378353e6 in os_overcommits_proc () at src/pages.c:475
+#16 _rjem_je_pages_boot () at src/pages.c:616
+#17 0x000055a53780ad0e in malloc_init_hard_a0_locked () at src/jemalloc.c:1518
+#18 0x000055a53780cf46 in malloc_init_hard () at src/jemalloc.c:1750
+#19 0x000055a5378072ee in malloc_init () at src/jemalloc.c:223
+#20 imalloc_init_check (sopts=<optimised out>, dopts=<optimised out>) at src/jemalloc.c:2229
+#21 imalloc (sopts=<optimised out>, dopts=<optimised out>) at src/jemalloc.c:2260
+#22 _rjem_je_malloc_default (size=72704) at src/jemalloc.c:2289
+#23 0x000055a537807630 in malloc (size=94168091620040, size@entry=72704) at include/jemalloc/internal/cache_bin.h:100
+#24 0x00007f36e623b950 in (anonymous namespace)::pool::pool (this=0x7f36e678cde0 <(anonymous namespace)::emergency_pool>)
+at ../../../../gcc-5.5.0/libstdc++-v3/libsupc++/eh_alloc.cc:117
+#25 __static_initialization_and_destruction_0 (__priority=65535, __initialize_p=1) at ../../../../gcc-5.5.0/libstdc++-v3/libsupc++/eh_alloc.cc:244
+#26 _GLOBAL__sub_I_eh_alloc.cc(void) () at ../../../../gcc-5.5.0/libstdc++-v3/libsupc++/eh_alloc.cc:307
+#27 0x00007f36eac28e5e in call_init () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/ld-linux-x86-64.so.2
+#28 0x00007f36eac28f4c in _dl_init () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/ld-linux-x86-64.so.2
+#29 0x00007f36eac407ca in _dl_start_user () from /media/build1/poky/build/tmp-musl/sysroots-uninative/x86_64-linux/lib/ld-linux-x86-64.so.2
+
+
+Index: git/makewrappers
+===================================================================
+--- git.orig/makewrappers
++++ git/makewrappers
+@@ -232,6 +232,7 @@ class Function:
+ self.fd_arg = False
+ self.noignore_path = False
+ self.hand_wrapped = None
++ self.disabled_condition = None
+ self.async_skip = None
+ # used for the copyright date when creating stub functions
+ self.date = datetime.date.today().year
+@@ -340,6 +341,13 @@ class Function:
+ else:
+ return ""
+
++ def extra_disabled_condition(self):
++ if self.disabled_condition:
++ if self.name == "syscall":
++ return 'pseudo_util_initted == -1'
++ return 'pseudo_util_initted == -1 && strstr(path, "malloc.conf") != NULL'
++ return "0"
++
+ def comment(self):
+ """declare self (in a comment)"""
+ return self.decl(comment = True)
+Index: git/templates/wrapfuncs.c
+===================================================================
+--- git.orig/templates/wrapfuncs.c
++++ git/templates/wrapfuncs.c
+@@ -26,6 +26,16 @@ ${name}(${decl_args}) {
+
+ ${maybe_async_skip}
+
++ if (${extra_disabled_condition}) {
++ ${variadic_start}
++ ${type} (*f)(${decl_args}) = (${type} (*)(${decl_args})) NULL;
++ f = dlsym(RTLD_NEXT, "${name}");
++ ${rc_assign} f(${call_args});
++ ${variadic_end}
++ PROFILE_DONE;
++ ${rc_return}
++ }
++
+ if (!pseudo_check_wrappers() || !real_$name) {
+ /* rc was initialized to the "failure" value */
+ pseudo_enosys("${name}");
+Index: git/ports/unix/wrapfuncs.in
+===================================================================
+--- git.orig/ports/unix/wrapfuncs.in
++++ git/ports/unix/wrapfuncs.in
+@@ -59,8 +59,8 @@ int symlinkat(const char *oldname, int d
+ int unlink(const char *path); /* flags=AT_SYMLINK_NOFOLLOW */
+ int unlinkat(int dirfd, const char *path, int rflags); /* flags=AT_SYMLINK_NOFOLLOW */
+ # primarily for use with chroot()
+-ssize_t readlink(const char *path, char *buf, size_t bufsiz); /* flags=AT_SYMLINK_NOFOLLOW */
+-ssize_t readlinkat(int dirfd, const char *path, char *buf, size_t bufsiz); /* flags=AT_SYMLINK_NOFOLLOW */
++ssize_t readlink(const char *path, char *buf, size_t bufsiz); /* flags=AT_SYMLINK_NOFOLLOW, disabled_condition=1 */
++ssize_t readlinkat(int dirfd, const char *path, char *buf, size_t bufsiz); /* flags=AT_SYMLINK_NOFOLLOW, disabled_condition=1 */
+ int system(const char *command);
+ FILE *popen(const char *command, const char *mode); /* hand_wrapped=1 */
+ # Based on experiments by Richard Purdie: Allow pseudo to eliminate
+Index: git/pseudo_util.c
+===================================================================
+--- git.orig/pseudo_util.c
++++ git/pseudo_util.c
+@@ -98,7 +98,7 @@ static unsigned long pseudo_debug_flags_
+ * program starts playing with things, so we need to do our
+ * best to handle that case.
+ */
+-static int pseudo_util_initted = -1; /* Not yet run */
++int pseudo_util_initted = -1; /* Not yet run */
+
+ /* bypass wrapper logic on path computations */
+ int (*pseudo_real_lstat)(const char *path, PSEUDO_STATBUF *buf) = NULL;
+Index: git/pseudo_client.h
+===================================================================
+--- git.orig/pseudo_client.h
++++ git/pseudo_client.h
+@@ -49,6 +49,7 @@ extern int pseudo_pwd_lck_open(void);
+ extern int pseudo_pwd_lck_close(void);
+ extern FILE *pseudo_pwd;
+ extern FILE *pseudo_grp;
++extern int pseudo_util_initted;
+
+ /* pseudo_wrappers will try to initialize these */
+ extern int (*pseudo_real_lstat)(const char *path, PSEUDO_STATBUF *buf);
+Index: git/ports/linux/wrapfuncs.in
+===================================================================
+--- git.orig/ports/linux/wrapfuncs.in
++++ git/ports/linux/wrapfuncs.in
+@@ -59,7 +59,7 @@ int getpw(uid_t uid, char *buf);
+ int getpwent_r(struct passwd *pwbuf, char *buf, size_t buflen, struct passwd **pwbufp);
+ int getgrent_r(struct group *gbuf, char *buf, size_t buflen, struct group **gbufp);
+ int capset(cap_user_header_t hdrp, const cap_user_data_t datap); /* real_func=pseudo_capset */
+-long syscall(long nr, ...); /* hand_wrapped=1 */
++long syscall(long nr, ...); /* hand_wrapped=1, disabled_condition=1 */
+ int renameat2(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags); /* flags=AT_SYMLINK_NOFOLLOW */
+ int prctl(int option, ...); /* hand_wrapped=1 */
+ int close_range(unsigned int lowfd, unsigned int maxfd, int flags);
+Index: git/ports/linux/pseudo_wrappers.c
+===================================================================
+--- git.orig/ports/linux/pseudo_wrappers.c
++++ git/ports/linux/pseudo_wrappers.c
+@@ -59,6 +59,13 @@ syscall(long number, ...) {
+ long rc = -1;
+ va_list ap;
+
++ if (pseudo_util_initted == -1) {
++ long (*f)(long nr, ...) = (long (*)(long nr, ...)) NULL;
++ f = dlsym(RTLD_NEXT, "syscall");
++ void *res = __builtin_apply((void (*)(void)) f, __builtin_apply_args(), sizeof(long) * 7);
++ __builtin_return(res);
++ }
++
+ if (!pseudo_check_wrappers() || !real_syscall) {
+ /* rc was initialized to the "failure" value */
+ pseudo_enosys("syscall");
@@ -2,6 +2,7 @@ require pseudo.inc
SRC_URI = "git://git.yoctoproject.org/pseudo;branch=oe-core;protocol=https \
file://0001-configure-Prune-PIE-flags.patch \
+ file://mallocignore.patch \
file://fallback-passwd \
file://fallback-group \
"
We see hangs if we leave pseudo enabled when running rust do_install. The prebuilt rust binaries use jemalloc as a memory allocator and it is interacting rather badly with pseudo's LD_PRELOAD. The hangs are because pseudo intercepts are called at malloc init time but that triggers pseudo's own init code which calls malloc itself and it deadlocks. There are two issue entry points. One is that it calls readlink() or readlinkat() on /etc/malloc.conf to load it's configuration. It also calls syscall() instead of open() on some /proc files, ironically, to avoid wrapper libs like ours, except we also wrap syscall(). This patch hacks pseudo to break the deadlocks in those functions by assuming a library won't make a readlink() call for a file containing "malloc.conf" or call syscall() as it's first library call. Sending as an RFC so people can understand why rust deadlocks and see if anyone has any better ideas on how to fix this. We may just decide not to run rust under pseudo instead. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- .../pseudo/files/mallocignore.patch | 183 +++++++++++++++++++++ meta/recipes-devtools/pseudo/pseudo_git.bb | 1 + 2 files changed, 184 insertions(+) create mode 100644 meta/recipes-devtools/pseudo/files/mallocignore.patch