| Message ID | bb3ae8dbd929eac83394be5fe3284dd79c895e44.1642662353.git.liezhi.yang@windriver.com |
|---|---|
| State | New |
| Headers | show |
| Series | [1/1] bitbake.conf: Add BB_TASK_NETWORK to enable task network globally | expand |
Hi Robert, On 1/20/22 08:09, Robert Yang wrote: > The NIS or icecc can't work when task network is dissable, add BB_TASK_NETWORK > to enable network globally for such exceptions. > > Note, enable nscd on the build machine might be a solution, but that isn't > reliable since it depends on whether the network function has been cached or > not. > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > --- > meta/classes/icecc.bbclass | 2 ++ > meta/conf/bitbake.conf | 3 +++ > meta/lib/oe/utils.py | 15 +++++++++++++++ > 3 files changed, 20 insertions(+) > > diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass > index 794e9930ad9..c39c86458a4 100644 > --- a/meta/classes/icecc.bbclass > +++ b/meta/classes/icecc.bbclass > @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" > > HOSTTOOLS_NONFATAL += "icecc patchelf" > > +BB_TASK_NETWORK ? = "1" > + > # This version can be incremented when changes are made to the environment that > # invalidate the version on the compile nodes. Changing it will cause a new > # environment to be created. > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index fba99e8f0cd..bf5bcd55519 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" > # what it would be anyway if the signature generator (e.g. OEEquivHash) doesn't > # support unihashes. > BB_UNIHASH ?= "${BB_TASKHASH}" > + > +# Enable task network for remote user such as NIS. > +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" > diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py > index 136650e6f74..c21f034aafc 100644 > --- a/meta/lib/oe/utils.py > +++ b/meta/lib/oe/utils.py > @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): > total += sum(roundup(getsize(os.path.join(root, name))) for name in files) > total += roundup(getsize(root)) > return total > + > +def is_local_uid(uid=''): > + """ > + Check whether uid is a local one or not. > + Can't use pwd module since it gets all UIDs, not local ones only. > + """ > + if not uid: > + uid = os.getuid() > + local_uids = set() > + with open('/etc/passwd', 'r') as f: > + for line in f.readlines(): > + if not ':' in line: > + continue > + local_uids.add(line.split(':')[2]) > + return uid in local_uids > You can return ASAP instead of iterating over all users: with open('/etc/passwd', 'r') as f: for line in f.readlines(): if not ':' in line: continue if line.split(':')[2] == uid: return True return False Otherwise, since we explicitly add icecc functions to existing tasks, can't you just add [network] = "1" to all those tasks in icecc.bbclass? e.g. do_configure[network] = "1" ? Also, even if icecc is globally inherited and enabled, recipes can still disable its use on a per-recipe basis in which case I don't think we should have the network enabled? I can't comment on NIS support. Cheers, Quentin
On Wed, 2022-01-19 at 23:09 -0800, Robert Yang wrote: > The NIS or icecc can't work when task network is dissable, add BB_TASK_NETWORK > to enable network globally for such exceptions. > > Note, enable nscd on the build machine might be a solution, but that isn't > reliable since it depends on whether the network function has been cached or > not. > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > --- > meta/classes/icecc.bbclass | 2 ++ > meta/conf/bitbake.conf | 3 +++ > meta/lib/oe/utils.py | 15 +++++++++++++++ > 3 files changed, 20 insertions(+) > > diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass > index 794e9930ad9..c39c86458a4 100644 > --- a/meta/classes/icecc.bbclass > +++ b/meta/classes/icecc.bbclass > @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" > > HOSTTOOLS_NONFATAL += "icecc patchelf" > > +BB_TASK_NETWORK ? = "1" > + As Quentin mentions, I'd much rather this was done on the tasks using icecc using the flag. > # This version can be incremented when changes are made to the environment that > # invalidate the version on the compile nodes. Changing it will cause a new > # environment to be created. > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index fba99e8f0cd..bf5bcd55519 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" > # what it would be anyway if the signature generator (e.g. OEEquivHash) doesn't > # support unihashes. > BB_UNIHASH ?= "${BB_TASKHASH}" > + > +# Enable task network for remote user such as NIS. > +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" > diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py > index 136650e6f74..c21f034aafc 100644 > --- a/meta/lib/oe/utils.py > +++ b/meta/lib/oe/utils.py > @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): > total += sum(roundup(getsize(os.path.join(root, name))) for name in files) > total += roundup(getsize(root)) > return total > + > +def is_local_uid(uid=''): > + """ > + Check whether uid is a local one or not. > + Can't use pwd module since it gets all UIDs, not local ones only. > + """ > + if not uid: > + uid = os.getuid() > + local_uids = set() > + with open('/etc/passwd', 'r') as f: > + for line in f.readlines(): > + if not ':' in line: > + continue > + local_uids.add(line.split(':')[2]) > + return uid in local_uids I think we should move this code into bitbake as I really dislike having it in bitbake.conf and that [0][1] syntax is horrible too... I'm also not convinced about the name "BB_TASK_NETWORK" and think we need something better. If we're moving the code to bitbake we might not need a name right now though. Cheers, Richard
> -----Original Message----- > From: openembedded-core@lists.openembedded.org <openembedded- > core@lists.openembedded.org> On Behalf Of Robert Yang > Sent: den 20 januari 2022 08:09 > To: openembedded-core@lists.openembedded.org > Cc: jupiter.hce@gmail.com > Subject: [OE-core] [PATCH 1/1] bitbake.conf: Add BB_TASK_NETWORK to enable > task network globally > > The NIS or icecc can't work when task network is dissable, add > BB_TASK_NETWORK to enable network globally for such exceptions. > > Note, enable nscd on the build machine might be a solution, but that isn't > reliable since it depends on whether the network function has been cached > or not. > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > --- > meta/classes/icecc.bbclass | 2 ++ > meta/conf/bitbake.conf | 3 +++ > meta/lib/oe/utils.py | 15 +++++++++++++++ > 3 files changed, 20 insertions(+) > > diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass > index 794e9930ad9..c39c86458a4 100644 > --- a/meta/classes/icecc.bbclass > +++ b/meta/classes/icecc.bbclass > @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" > > HOSTTOOLS_NONFATAL += "icecc patchelf" > > +BB_TASK_NETWORK ? = "1" > + > # This version can be incremented when changes are made to the environment that > # invalidate the version on the compile nodes. Changing it will cause a new > # environment to be created. > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index fba99e8f0cd..bf5bcd55519 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" > # what it would be anyway if the signature generator (e.g. OEEquivHash) > doesn't > # support unihashes. > BB_UNIHASH ?= "${BB_TASKHASH}" > + > +# Enable task network for remote user such as NIS. > +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" I think this is more readable: BB_TASK_NETWORK ??= "${@0 if oe.utils.is_local_uid() else 1}" An even more readable solution would be to rename the function to is_nonlocal_uid() and make it return the inverse of what it returns now. You probably also want to change it to use := instead of ??= so that /etc/passwd is only parsed once. This might require to introduce another variable: UID_IS_NONLOCAL := "${@0 if oe.utils.is_local_uid() else 1}" BB_TASK_NETWORK ??= "${UID_IS_NONLOCAL}" > diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py > index 136650e6f74..c21f034aafc 100644 > --- a/meta/lib/oe/utils.py > +++ b/meta/lib/oe/utils.py > @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): > total += sum(roundup(getsize(os.path.join(root, name))) for name in files) > total += roundup(getsize(root)) > return total > + > +def is_local_uid(uid=''): > + """ > + Check whether uid is a local one or not. > + Can't use pwd module since it gets all UIDs, not local ones only. > + """ > + if not uid: > + uid = os.getuid() > + local_uids = set() > + with open('/etc/passwd', 'r') as f: > + for line in f.readlines(): > + if not ':' in line: > + continue > + local_uids.add(line.split(':')[2]) > + return uid in local_uids > -- > 2.31.1 //Peter
On 1/20/22 9:31 PM, Quentin Schulz wrote: > Hi Robert, > > On 1/20/22 08:09, Robert Yang wrote: >> The NIS or icecc can't work when task network is dissable, add BB_TASK_NETWORK >> to enable network globally for such exceptions. >> >> Note, enable nscd on the build machine might be a solution, but that isn't >> reliable since it depends on whether the network function has been cached or >> not. >> >> Signed-off-by: Robert Yang <liezhi.yang@windriver.com> >> --- >> meta/classes/icecc.bbclass | 2 ++ >> meta/conf/bitbake.conf | 3 +++ >> meta/lib/oe/utils.py | 15 +++++++++++++++ >> 3 files changed, 20 insertions(+) >> >> diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass >> index 794e9930ad9..c39c86458a4 100644 >> --- a/meta/classes/icecc.bbclass >> +++ b/meta/classes/icecc.bbclass >> @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" >> HOSTTOOLS_NONFATAL += "icecc patchelf" >> +BB_TASK_NETWORK ? = "1" >> + >> # This version can be incremented when changes are made to the environment that >> # invalidate the version on the compile nodes. Changing it will cause a new >> # environment to be created. >> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf >> index fba99e8f0cd..bf5bcd55519 100644 >> --- a/meta/conf/bitbake.conf >> +++ b/meta/conf/bitbake.conf >> @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" >> # what it would be anyway if the signature generator (e.g. OEEquivHash) doesn't >> # support unihashes. >> BB_UNIHASH ?= "${BB_TASKHASH}" >> + >> +# Enable task network for remote user such as NIS. >> +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" >> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py >> index 136650e6f74..c21f034aafc 100644 >> --- a/meta/lib/oe/utils.py >> +++ b/meta/lib/oe/utils.py >> @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): >> total += sum(roundup(getsize(os.path.join(root, name))) for name in >> files) >> total += roundup(getsize(root)) >> return total >> + >> +def is_local_uid(uid=''): >> + """ >> + Check whether uid is a local one or not. >> + Can't use pwd module since it gets all UIDs, not local ones only. >> + """ >> + if not uid: >> + uid = os.getuid() >> + local_uids = set() >> + with open('/etc/passwd', 'r') as f: >> + for line in f.readlines(): >> + if not ':' in line: >> + continue >> + local_uids.add(line.split(':')[2]) >> + return uid in local_uids >> > > You can return ASAP instead of iterating over all users: > > with open('/etc/passwd', 'r') as f: > for line in f.readlines(): > if not ':' in line: > continue > if line.split(':')[2] == uid: > return True > return False Yes, you're right, I renamed the func from get_all_local_uids to is_local_uid(), and forgot to change the logical. > > Otherwise, since we explicitly add icecc functions to existing tasks, can't you > just add [network] = "1" to all those tasks in icecc.bbclass? e.g. > do_configure[network] = "1" ? The icecc issue is reported by Jose, I'm not sure whether that works for him or not. > > Also, even if icecc is globally inherited and enabled, recipes can still disable > its use on a per-recipe basis in which case I don't think we should have the > network enabled? The network is disabled by default unless set task[network] = "1", and BB_TASK_NETWORK is for preventing the disable, so you can't disable network per-recipe basis, otherwise, it won't work for the case like NIS. Maybe a better name should be FORBIDDEN_DISABLE_TASK_NETWORK. // Robert > > I can't comment on NIS support. > > Cheers, > Quentin
On 1/20/22 9:38 PM, Richard Purdie wrote: > On Wed, 2022-01-19 at 23:09 -0800, Robert Yang wrote: >> The NIS or icecc can't work when task network is dissable, add BB_TASK_NETWORK >> to enable network globally for such exceptions. >> >> Note, enable nscd on the build machine might be a solution, but that isn't >> reliable since it depends on whether the network function has been cached or >> not. >> >> Signed-off-by: Robert Yang <liezhi.yang@windriver.com> >> --- >> meta/classes/icecc.bbclass | 2 ++ >> meta/conf/bitbake.conf | 3 +++ >> meta/lib/oe/utils.py | 15 +++++++++++++++ >> 3 files changed, 20 insertions(+) >> >> diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass >> index 794e9930ad9..c39c86458a4 100644 >> --- a/meta/classes/icecc.bbclass >> +++ b/meta/classes/icecc.bbclass >> @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" >> >> HOSTTOOLS_NONFATAL += "icecc patchelf" >> >> +BB_TASK_NETWORK ? = "1" >> + > > As Quentin mentions, I'd much rather this was done on the tasks using icecc > using the flag. > > >> # This version can be incremented when changes are made to the environment that >> # invalidate the version on the compile nodes. Changing it will cause a new >> # environment to be created. >> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf >> index fba99e8f0cd..bf5bcd55519 100644 >> --- a/meta/conf/bitbake.conf >> +++ b/meta/conf/bitbake.conf >> @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" >> # what it would be anyway if the signature generator (e.g. OEEquivHash) doesn't >> # support unihashes. >> BB_UNIHASH ?= "${BB_TASKHASH}" >> + >> +# Enable task network for remote user such as NIS. >> +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" >> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py >> index 136650e6f74..c21f034aafc 100644 >> --- a/meta/lib/oe/utils.py >> +++ b/meta/lib/oe/utils.py >> @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): >> total += sum(roundup(getsize(os.path.join(root, name))) for name in files) >> total += roundup(getsize(root)) >> return total >> + >> +def is_local_uid(uid=''): >> + """ >> + Check whether uid is a local one or not. >> + Can't use pwd module since it gets all UIDs, not local ones only. >> + """ >> + if not uid: >> + uid = os.getuid() >> + local_uids = set() >> + with open('/etc/passwd', 'r') as f: >> + for line in f.readlines(): >> + if not ':' in line: >> + continue >> + local_uids.add(line.split(':')[2]) >> + return uid in local_uids > > I think we should move this code into bitbake as I really dislike having it in > bitbake.conf and that [0][1] syntax is horrible too... > > I'm also not convinced about the name "BB_TASK_NETWORK" and think we need > something better. If we're moving the code to bitbake we might not need a name > right now though. Yes, it's enough for NIS without a name, it seems that NIS is the only case atm, I will send a V2 about this. // Robert > > Cheers, > > Richard > > >
On 1/20/22 9:47 PM, Peter Kjellerstedt wrote: >> -----Original Message----- >> From: openembedded-core@lists.openembedded.org <openembedded- >> core@lists.openembedded.org> On Behalf Of Robert Yang >> Sent: den 20 januari 2022 08:09 >> To: openembedded-core@lists.openembedded.org >> Cc: jupiter.hce@gmail.com >> Subject: [OE-core] [PATCH 1/1] bitbake.conf: Add BB_TASK_NETWORK to enable >> task network globally >> >> The NIS or icecc can't work when task network is dissable, add >> BB_TASK_NETWORK to enable network globally for such exceptions. >> >> Note, enable nscd on the build machine might be a solution, but that isn't >> reliable since it depends on whether the network function has been cached >> or not. >> >> Signed-off-by: Robert Yang <liezhi.yang@windriver.com> >> --- >> meta/classes/icecc.bbclass | 2 ++ >> meta/conf/bitbake.conf | 3 +++ >> meta/lib/oe/utils.py | 15 +++++++++++++++ >> 3 files changed, 20 insertions(+) >> >> diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass >> index 794e9930ad9..c39c86458a4 100644 >> --- a/meta/classes/icecc.bbclass >> +++ b/meta/classes/icecc.bbclass >> @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" >> >> HOSTTOOLS_NONFATAL += "icecc patchelf" >> >> +BB_TASK_NETWORK ? = "1" >> + >> # This version can be incremented when changes are made to the environment that >> # invalidate the version on the compile nodes. Changing it will cause a new >> # environment to be created. >> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf >> index fba99e8f0cd..bf5bcd55519 100644 >> --- a/meta/conf/bitbake.conf >> +++ b/meta/conf/bitbake.conf >> @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" >> # what it would be anyway if the signature generator (e.g. OEEquivHash) >> doesn't >> # support unihashes. >> BB_UNIHASH ?= "${BB_TASKHASH}" >> + >> +# Enable task network for remote user such as NIS. >> +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" > > I think this is more readable: > > BB_TASK_NETWORK ??= "${@0 if oe.utils.is_local_uid() else 1}" > > An even more readable solution would be to rename the function to > is_nonlocal_uid() and make it return the inverse of what it returns > now. > > You probably also want to change it to use := instead of ??= so that > /etc/passwd is only parsed once. This might require to introduce > another variable: > > UID_IS_NONLOCAL := "${@0 if oe.utils.is_local_uid() else 1}" > BB_TASK_NETWORK ??= "${UID_IS_NONLOCAL}" We may not need this any more since NIS seems that is the only case which doesn't work, and can't be fixed (or not easy to fix) by set network per task. // Robert > >> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py >> index 136650e6f74..c21f034aafc 100644 >> --- a/meta/lib/oe/utils.py >> +++ b/meta/lib/oe/utils.py >> @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): >> total += sum(roundup(getsize(os.path.join(root, name))) for name in files) >> total += roundup(getsize(root)) >> return total >> + >> +def is_local_uid(uid=''): >> + """ >> + Check whether uid is a local one or not. >> + Can't use pwd module since it gets all UIDs, not local ones only. >> + """ >> + if not uid: >> + uid = os.getuid() >> + local_uids = set() >> + with open('/etc/passwd', 'r') as f: >> + for line in f.readlines(): >> + if not ':' in line: >> + continue >> + local_uids.add(line.split(':')[2]) >> + return uid in local_uids >> -- >> 2.31.1 > > //Peter >
diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass index 794e9930ad9..c39c86458a4 100644 --- a/meta/classes/icecc.bbclass +++ b/meta/classes/icecc.bbclass @@ -41,6 +41,8 @@ ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" HOSTTOOLS_NONFATAL += "icecc patchelf" +BB_TASK_NETWORK ? = "1" + # This version can be incremented when changes are made to the environment that # invalidate the version on the compile nodes. Changing it will cause a new # environment to be created. diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf index fba99e8f0cd..bf5bcd55519 100644 --- a/meta/conf/bitbake.conf +++ b/meta/conf/bitbake.conf @@ -946,3 +946,6 @@ MULTILIB_VARIANTS ??= "" # what it would be anyway if the signature generator (e.g. OEEquivHash) doesn't # support unihashes. BB_UNIHASH ?= "${BB_TASKHASH}" + +# Enable task network for remote user such as NIS. +BB_TASK_NETWORK ??= "${@['1', '0'][oe.utils.is_local_uid()]}" diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 136650e6f74..c21f034aafc 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -595,3 +595,18 @@ def directory_size(root, blocksize=4096): total += sum(roundup(getsize(os.path.join(root, name))) for name in files) total += roundup(getsize(root)) return total + +def is_local_uid(uid=''): + """ + Check whether uid is a local one or not. + Can't use pwd module since it gets all UIDs, not local ones only. + """ + if not uid: + uid = os.getuid() + local_uids = set() + with open('/etc/passwd', 'r') as f: + for line in f.readlines(): + if not ':' in line: + continue + local_uids.add(line.split(':')[2]) + return uid in local_uids
The NIS or icecc can't work when task network is dissable, add BB_TASK_NETWORK to enable network globally for such exceptions. Note, enable nscd on the build machine might be a solution, but that isn't reliable since it depends on whether the network function has been cached or not. Signed-off-by: Robert Yang <liezhi.yang@windriver.com> --- meta/classes/icecc.bbclass | 2 ++ meta/conf/bitbake.conf | 3 +++ meta/lib/oe/utils.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+)