summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xbitbake/bin/bitbake-setup94
-rwxr-xr-xbitbake/bin/toaster21
-rw-r--r--bitbake/lib/bb/tests/setup.py60
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py99
4 files changed, 166 insertions, 108 deletions
diff --git a/bitbake/bin/bitbake-setup b/bitbake/bin/bitbake-setup
index a0426005ff..3cd67805fe 100755
--- a/bitbake/bin/bitbake-setup
+++ b/bitbake/bin/bitbake-setup
@@ -114,7 +114,7 @@ def checkout_layers(layers, layerdir, d):
114 os.remove(symlink) 114 os.remove(symlink)
115 os.symlink(os.path.relpath(t,layerdir),symlink) 115 os.symlink(os.path.relpath(t,layerdir),symlink)
116 116
117def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir): 117def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir):
118 def _setup_build_conf(layers, build_conf_dir): 118 def _setup_build_conf(layers, build_conf_dir):
119 os.makedirs(build_conf_dir) 119 os.makedirs(build_conf_dir)
120 layers_s = [] 120 layers_s = []
@@ -188,7 +188,7 @@ def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
188 f.write('\n') 188 f.write('\n')
189 f.write(content) 189 f.write(content)
190 190
191 bitbake_builddir = os.path.join(builddir, "build") 191 bitbake_builddir = os.path.join(setupdir, "build")
192 print("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir)) 192 print("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir))
193 193
194 template = bitbake_config.get("oe-template") 194 template = bitbake_config.get("oe-template")
@@ -220,7 +220,7 @@ def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
220 _prepend_passthrough_to_init_build_env(bitbake_builddir) 220 _prepend_passthrough_to_init_build_env(bitbake_builddir)
221 221
222 siteconf_symlink = os.path.join(bitbake_confdir, "site.conf") 222 siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
223 siteconf = os.path.normpath(os.path.join(builddir, '..', "site.conf")) 223 siteconf = os.path.normpath(os.path.join(setupdir, '..', "site.conf"))
224 if os.path.lexists(siteconf_symlink): 224 if os.path.lexists(siteconf_symlink):
225 os.remove(symlink) 225 os.remove(symlink)
226 os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink) 226 os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)
@@ -267,7 +267,7 @@ def get_registry_config(registry_path, id):
267 return os.path.join(root, f) 267 return os.path.join(root, f)
268 raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id)) 268 raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))
269 269
270def update_build(config, confdir, builddir, layerdir, d): 270def update_build(config, confdir, setupdir, layerdir, d):
271 layer_config = config["data"]["sources"] 271 layer_config = config["data"]["sources"]
272 layer_overrides = config["source-overrides"]["sources"] 272 layer_overrides = config["source-overrides"]["sources"]
273 for k,v in layer_overrides.items(): 273 for k,v in layer_overrides.items():
@@ -276,7 +276,7 @@ def update_build(config, confdir, builddir, layerdir, d):
276 checkout_layers(layer_config, layerdir, d) 276 checkout_layers(layer_config, layerdir, d)
277 bitbake_config = config["bitbake-config"] 277 bitbake_config = config["bitbake-config"]
278 thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None 278 thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
279 setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir) 279 setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir)
280 280
281def int_input(allowed_values): 281def int_input(allowed_values):
282 n = None 282 n = None
@@ -441,22 +441,22 @@ def init_config(top_dir, settings, args, d):
441 upstream_config = obtain_config(top_dir, settings, args, source_overrides, d) 441 upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
442 print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options']))) 442 print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))
443 443
444 builddir = os.path.join(os.path.abspath(top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_"))) 444 setupdir = os.path.join(os.path.abspath(top_dir), args.setup_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
445 if os.path.exists(os.path.join(builddir, "layers")): 445 if os.path.exists(os.path.join(setupdir, "layers")):
446 print(f"Build already initialized in:\n {builddir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize a build in this directory, remove it, and run 'bitbake-setup init' again.") 446 print(f"Setup already initialized in:\n {setupdir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize in this directory, remove it, and run 'bitbake-setup init' again.")
447 return 447 return
448 448
449 print("Initializing a build in\n {}".format(builddir)) 449 print("Initializing a setup directory in\n {}".format(setupdir))
450 if not args.non_interactive: 450 if not args.non_interactive:
451 y_or_n = input('Continue? (y/N): ') 451 y_or_n = input('Continue? (y/N): ')
452 if y_or_n != 'y': 452 if y_or_n != 'y':
453 exit() 453 exit()
454 print() 454 print()
455 455
456 os.makedirs(builddir, exist_ok=True) 456 os.makedirs(setupdir, exist_ok=True)
457 457
458 confdir = os.path.join(builddir, "config") 458 confdir = os.path.join(setupdir, "config")
459 layerdir = os.path.join(builddir, "layers") 459 layerdir = os.path.join(setupdir, "layers")
460 460
461 os.makedirs(confdir) 461 os.makedirs(confdir)
462 os.makedirs(layerdir) 462 os.makedirs(layerdir)
@@ -471,7 +471,7 @@ def init_config(top_dir, settings, args, d):
471 471
472 write_config(upstream_config, confdir) 472 write_config(upstream_config, confdir)
473 commit_config(confdir) 473 commit_config(confdir)
474 update_build(upstream_config, confdir, builddir, layerdir, d) 474 update_build(upstream_config, confdir, setupdir, layerdir, d)
475 475
476 bb.event.remove("bb.build.TaskProgress", None) 476 bb.event.remove("bb.build.TaskProgress", None)
477 477
@@ -513,10 +513,10 @@ def are_layers_changed(layers, layerdir, d):
513 return changed 513 return changed
514 514
515def build_status(top_dir, settings, args, d, update=False): 515def build_status(top_dir, settings, args, d, update=False):
516 builddir = args.build_dir 516 setupdir = args.setup_dir
517 517
518 confdir = os.path.join(builddir, "config") 518 confdir = os.path.join(setupdir, "config")
519 layerdir = os.path.join(builddir, "layers") 519 layerdir = os.path.join(setupdir, "layers")
520 520
521 current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json"))) 521 current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))
522 522
@@ -530,20 +530,20 @@ def build_status(top_dir, settings, args, d, update=False):
530 config_diff = bb.process.run('git -C {} diff'.format(confdir))[0] 530 config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
531 531
532 if config_diff: 532 if config_diff:
533 print('\nConfiguration in {} has changed:\n{}'.format(builddir, config_diff)) 533 print('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff))
534 if update: 534 if update:
535 commit_config(confdir) 535 commit_config(confdir)
536 update_build(new_upstream_config, confdir, builddir, layerdir, d) 536 update_build(new_upstream_config, confdir, setupdir, layerdir, d)
537 else: 537 else:
538 bb.process.run('git -C {} restore config-upstream.json'.format(confdir)) 538 bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
539 return 539 return
540 540
541 if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d): 541 if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
542 if update: 542 if update:
543 update_build(current_upstream_config, confdir, builddir, layerdir, d) 543 update_build(current_upstream_config, confdir, setupdir, layerdir, d)
544 return 544 return
545 545
546 print("\nConfiguration in {} has not changed.".format(builddir)) 546 print("\nConfiguration in {} has not changed.".format(setupdir))
547 547
548def build_update(top_dir, settings, args, d): 548def build_update(top_dir, settings, args, d):
549 build_status(top_dir, settings, args, d, update=True) 549 build_status(top_dir, settings, args, d, update=True)
@@ -616,7 +616,7 @@ def list_configs(top_dir, settings, args, d):
616 print("Available configurations written into {}".format(args.write_json)) 616 print("Available configurations written into {}".format(args.write_json))
617 617
618def install_buildtools(top_dir, settings, args, d): 618def install_buildtools(top_dir, settings, args, d):
619 buildtools_install_dir = os.path.join(args.build_dir, 'buildtools') 619 buildtools_install_dir = os.path.join(args.setup_dir, 'buildtools')
620 if os.path.exists(buildtools_install_dir): 620 if os.path.exists(buildtools_install_dir):
621 if not args.force: 621 if not args.force:
622 print("Buildtools are already installed in {}.".format(buildtools_install_dir)) 622 print("Buildtools are already installed in {}.".format(buildtools_install_dir))
@@ -629,8 +629,8 @@ def install_buildtools(top_dir, settings, args, d):
629 return 629 return
630 shutil.rmtree(buildtools_install_dir) 630 shutil.rmtree(buildtools_install_dir)
631 631
632 install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools') 632 install_buildtools = os.path.join(args.setup_dir, 'layers/oe-scripts/install-buildtools')
633 buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S"))) 633 buildtools_download_dir = os.path.join(args.setup_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
634 print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir)) 634 print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
635 subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True) 635 subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)
636 636
@@ -696,24 +696,24 @@ def settings_func(top_dir, all_settings, args):
696 elif args.subcommand == 'set' or args.subcommand == 'unset': 696 elif args.subcommand == 'set' or args.subcommand == 'unset':
697 change_setting(top_dir, args) 697 change_setting(top_dir, args)
698 698
699def get_build_dir_via_bbpath(): 699def get_setup_dir_via_bbpath():
700 bbpath = os.environ.get('BBPATH') 700 bbpath = os.environ.get('BBPATH')
701 if bbpath: 701 if bbpath:
702 bitbake_dir = os.path.normpath(bbpath.split(':')[0]) 702 bitbake_dir = os.path.normpath(bbpath.split(':')[0])
703 if os.path.exists(os.path.join(bitbake_dir,'init-build-env')): 703 if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
704 build_dir = os.path.dirname(bitbake_dir) 704 setup_dir = os.path.dirname(bitbake_dir)
705 return build_dir 705 return setup_dir
706 return None 706 return None
707 707
708def get_top_dir(args, settings): 708def get_top_dir(args, settings):
709 build_dir_via_bbpath = get_build_dir_via_bbpath() 709 setup_dir_via_bbpath = get_setup_dir_via_bbpath()
710 if build_dir_via_bbpath: 710 if setup_dir_via_bbpath:
711 top_dir = os.path.dirname(build_dir_via_bbpath) 711 top_dir = os.path.dirname(setup_dir_via_bbpath)
712 if os.path.exists(default_settings_path(top_dir)): 712 if os.path.exists(default_settings_path(top_dir)):
713 return top_dir 713 return top_dir
714 714
715 if hasattr(args, 'build_dir'): 715 if hasattr(args, 'setup_dir'):
716 top_dir = os.path.dirname(os.path.normpath(args.build_dir)) 716 top_dir = os.path.dirname(os.path.normpath(args.setup_dir))
717 return top_dir 717 return top_dir
718 718
719 top_dir_prefix = settings['default']['top-dir-prefix'] 719 top_dir_prefix = settings['default']['top-dir-prefix']
@@ -734,12 +734,12 @@ def merge_settings(builtin_settings, global_settings, local_settings, cmdline_se
734 return all_settings 734 return all_settings
735 735
736def main(): 736def main():
737 def add_build_dir_arg(parser): 737 def add_setup_dir_arg(parser):
738 build_dir = get_build_dir_via_bbpath() 738 setup_dir = get_setup_dir_via_bbpath()
739 if build_dir: 739 if setup_dir:
740 parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH") 740 parser.add_argument('--setup-dir', default=setup_dir, help="Path to the setup, default is %(default)s via BBPATH")
741 else: 741 else:
742 parser.add_argument('--build-dir', required=True, help="Path to the build") 742 parser.add_argument('--setup-dir', required=True, help="Path to the setup")
743 743
744 parser = argparse.ArgumentParser( 744 parser = argparse.ArgumentParser(
745 description="BitBake setup utility. Run with 'init' argument to get started.", 745 description="BitBake setup utility. Run with 'init' argument to get started.",
@@ -761,24 +761,24 @@ def main():
761 parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.') 761 parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
762 parser_list.set_defaults(func=list_configs) 762 parser_list.set_defaults(func=list_configs)
763 763
764 parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it') 764 parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a setup from it')
765 parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.") 765 parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
766 parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.') 766 parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
767 parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.') 767 parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
768 parser_init.add_argument('--build-dir-name', action='store', help='A custom build directory name under the top directory.') 768 parser_init.add_argument('--setup-dir-name', action='store', help='A custom setup directory name under the top directory.')
769 parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.') 769 parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
770 parser_init.set_defaults(func=init_config) 770 parser_init.set_defaults(func=init_config)
771 771
772 parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration') 772 parser_status = subparsers.add_parser('status', help='Check if the setup needs to be synchronized with configuration')
773 add_build_dir_arg(parser_status) 773 add_setup_dir_arg(parser_status)
774 parser_status.set_defaults(func=build_status) 774 parser_status.set_defaults(func=build_status)
775 775
776 parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration') 776 parser_update = subparsers.add_parser('update', help='Update a setup to be in sync with configuration')
777 add_build_dir_arg(parser_update) 777 add_setup_dir_arg(parser_update)
778 parser_update.set_defaults(func=build_update) 778 parser_update.set_defaults(func=build_update)
779 779
780 parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine') 780 parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
781 add_build_dir_arg(parser_install_buildtools) 781 add_setup_dir_arg(parser_install_buildtools)
782 parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.') 782 parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
783 parser_install_buildtools.set_defaults(func=install_buildtools) 783 parser_install_buildtools.set_defaults(func=install_buildtools)
784 784
@@ -786,7 +786,7 @@ def main():
786 parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory") 786 parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory")
787 787
788 parser_settings = subparsers.add_parser('settings', 788 parser_settings = subparsers.add_parser('settings',
789 help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of build configuration registry, downloads directory and other settings specific to a top directory)') 789 help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of configuration registry, downloads directory and other settings specific to a top directory)')
790 parser_settings.set_defaults(func=settings_func) 790 parser_settings.set_defaults(func=settings_func)
791 791
792 subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file") 792 subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file")
@@ -820,9 +820,9 @@ def main():
820 level=logger.getEffectiveLevel()) 820 level=logger.getEffectiveLevel())
821 821
822 if 'func' in args: 822 if 'func' in args:
823 if hasattr(args, 'build_dir'): 823 if hasattr(args, 'setup_dir'):
824 if not os.path.exists(os.path.join(args.build_dir,'build', 'init-build-env')): 824 if not os.path.exists(os.path.join(args.setup_dir,'build', 'init-build-env')):
825 print("Not a valid build directory: build/init-build-env does not exist in {}".format(args.build_dir)) 825 print("Not a valid setup directory: build/init-build-env does not exist in {}".format(args.setup_dir))
826 return 826 return
827 827
828 if not hasattr(args, 'non_interactive'): 828 if not hasattr(args, 'non_interactive'):
diff --git a/bitbake/bin/toaster b/bitbake/bin/toaster
index f002c8c159..d20cbe90ba 100755
--- a/bitbake/bin/toaster
+++ b/bitbake/bin/toaster
@@ -164,14 +164,20 @@ fi
164# in the local layers that currently make using an arbitrary 164# in the local layers that currently make using an arbitrary
165# toasterconf.json difficult. 165# toasterconf.json difficult.
166 166
167. $OE_ROOT/.templateconf 167# BBSETUP no longer supports .templateconf
168if [ -n "$TEMPLATECONF" ]; then 168if [ -f $OE_ROOT/.templateconf ] ; then
169 if [ ! -d "$TEMPLATECONF" ]; then 169 . $OE_ROOT/.templateconf
170 # Allow TEMPLATECONF=meta-xyz/conf as a shortcut 170 if [ -n "$TEMPLATECONF" ]; then
171 if [ -d "$OE_ROOT/$TEMPLATECONF" ]; then 171 if [ ! -d "$TEMPLATECONF" ]; then
172 TEMPLATECONF="$OE_ROOT/$TEMPLATECONF" 172 # Allow TEMPLATECONF=meta-xyz/conf as a shortcut
173 if [ -d "$OE_ROOT/$TEMPLATECONF" ]; then
174 TEMPLATECONF="$OE_ROOT/$TEMPLATECONF"
175 fi
173 fi 176 fi
174 fi 177 fi
178else
179 # Force poky
180 TEMPLATECONF="layers/meta-yocto/meta-poky/conf/templates/default"
175fi 181fi
176 182
177unset OE_ROOT 183unset OE_ROOT
@@ -255,7 +261,8 @@ fi
255# 2) the build dir (in build) 261# 2) the build dir (in build)
256# 3) the sqlite db if that is being used. 262# 3) the sqlite db if that is being used.
257# 4) pid's we need to clean up on exit/shutdown 263# 4) pid's we need to clean up on exit/shutdown
258export TOASTER_DIR=$TOASTERDIR 264# By default we move this all into the Toaster's parent build directory
265export TOASTER_DIR=$BUILDDIR
259export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS TOASTER_DIR" 266export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS TOASTER_DIR"
260 267
261# Determine the action. If specified by arguments, fine, if not, toggle it 268# Determine the action. If specified by arguments, fine, if not, toggle it
diff --git a/bitbake/lib/bb/tests/setup.py b/bitbake/lib/bb/tests/setup.py
index e320cdf56f..a17b8ac46a 100644
--- a/bitbake/lib/bb/tests/setup.py
+++ b/bitbake/lib/bb/tests/setup.py
@@ -109,19 +109,19 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
109 "configurations": [ 109 "configurations": [
110 { 110 {
111 "name": "gadget", 111 "name": "gadget",
112 "description": "Gadget build configuration", 112 "description": "Gadget configuration",
113 "oe-template": "test-configuration-gadget", 113 "oe-template": "test-configuration-gadget",
114 "oe-fragments": ["test-fragment-1"] 114 "oe-fragments": ["test-fragment-1"]
115 }, 115 },
116 { 116 {
117 "name": "gizmo", 117 "name": "gizmo",
118 "description": "Gizmo build configuration", 118 "description": "Gizmo configuration",
119 "oe-template": "test-configuration-gizmo", 119 "oe-template": "test-configuration-gizmo",
120 "oe-fragments": ["test-fragment-2"] 120 "oe-fragments": ["test-fragment-2"]
121 }, 121 },
122 { 122 {
123 "name": "gizmo-env-passthrough", 123 "name": "gizmo-env-passthrough",
124 "description": "Gizmo build configuration with environment-passthrough", 124 "description": "Gizmo configuration with environment-passthrough",
125 "bb-layers": ["layerC","layerD/meta-layer"], 125 "bb-layers": ["layerC","layerD/meta-layer"],
126 "oe-fragments": ["test-fragment-1"], 126 "oe-fragments": ["test-fragment-1"],
127 "bb-env-passthrough-additions": [ 127 "bb-env-passthrough-additions": [
@@ -132,24 +132,24 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
132 }, 132 },
133 { 133 {
134 "name": "gizmo-no-fragment", 134 "name": "gizmo-no-fragment",
135 "description": "Gizmo no-fragment template-only build configuration", 135 "description": "Gizmo no-fragment template-only configuration",
136 "oe-template": "test-configuration-gizmo" 136 "oe-template": "test-configuration-gizmo"
137 }, 137 },
138 { 138 {
139 "name": "gadget-notemplate", 139 "name": "gadget-notemplate",
140 "description": "Gadget notemplate build configuration", 140 "description": "Gadget notemplate configuration",
141 "bb-layers": ["layerA","layerB/meta-layer"], 141 "bb-layers": ["layerA","layerB/meta-layer"],
142 "oe-fragments": ["test-fragment-1"] 142 "oe-fragments": ["test-fragment-1"]
143 }, 143 },
144 { 144 {
145 "name": "gizmo-notemplate", 145 "name": "gizmo-notemplate",
146 "description": "Gizmo notemplate build configuration", 146 "description": "Gizmo notemplate configuration",
147 "bb-layers": ["layerC","layerD/meta-layer"], 147 "bb-layers": ["layerC","layerD/meta-layer"],
148 "oe-fragments": ["test-fragment-2"] 148 "oe-fragments": ["test-fragment-2"]
149 }, 149 },
150 { 150 {
151 "name": "gizmo-notemplate-with-thisdir", 151 "name": "gizmo-notemplate-with-thisdir",
152 "description": "Gizmo notemplate build configuration using THISDIR", 152 "description": "Gizmo notemplate configuration using THISDIR",
153 "bb-layers": ["layerC","layerD/meta-layer","{THISDIR}/layerE/meta-layer"], 153 "bb-layers": ["layerC","layerD/meta-layer","{THISDIR}/layerE/meta-layer"],
154 "oe-fragments": ["test-fragment-2"] 154 "oe-fragments": ["test-fragment-2"]
155 } 155 }
@@ -177,11 +177,11 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
177 self.git('add {}'.format(name), cwd=self.testrepopath) 177 self.git('add {}'.format(name), cwd=self.testrepopath)
178 self.git('commit -m "Adding {}"'.format(name), cwd=self.testrepopath) 178 self.git('commit -m "Adding {}"'.format(name), cwd=self.testrepopath)
179 179
180 def check_builddir_files(self, buildpath, test_file_content, json_config): 180 def check_setupdir_files(self, setuppath, test_file_content, json_config):
181 with open(os.path.join(buildpath, 'layers', 'test-repo', 'test-file')) as f: 181 with open(os.path.join(setuppath, 'layers', 'test-repo', 'test-file')) as f:
182 self.assertEqual(f.read(), test_file_content) 182 self.assertEqual(f.read(), test_file_content)
183 bitbake_config = json_config["bitbake-config"] 183 bitbake_config = json_config["bitbake-config"]
184 bb_build_path = os.path.join(buildpath, 'build') 184 bb_build_path = os.path.join(setuppath, 'build')
185 bb_conf_path = os.path.join(bb_build_path, 'conf') 185 bb_conf_path = os.path.join(bb_build_path, 'conf')
186 self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env'))) 186 self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env')))
187 187
@@ -203,7 +203,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
203 ) 203 )
204 self.assertIn(thisdir_layer, bblayers) 204 self.assertIn(thisdir_layer, bblayers)
205 else: 205 else:
206 self.assertIn(os.path.join(buildpath, "layers", l), bblayers) 206 self.assertIn(os.path.join(setuppath, "layers", l), bblayers)
207 207
208 if 'oe-fragment' in bitbake_config.keys(): 208 if 'oe-fragment' in bitbake_config.keys():
209 for f in bitbake_config["oe-fragments"]: 209 for f in bitbake_config["oe-fragments"]:
@@ -295,20 +295,20 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
295 for cf, v in test_configurations.items(): 295 for cf, v in test_configurations.items():
296 for c in v['buildconfigs']: 296 for c in v['buildconfigs']:
297 out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c)) 297 out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c))
298 buildpath = os.path.join(self.tempdir, 'bitbake-builds', '{}-{}'.format(cf, c)) 298 setuppath = os.path.join(self.tempdir, 'bitbake-builds', '{}-{}'.format(cf, c))
299 with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f: 299 with open(os.path.join(setuppath, 'config', "config-upstream.json")) as f:
300 config_upstream = json.load(f) 300 config_upstream = json.load(f)
301 self.check_builddir_files(buildpath, test_file_content, config_upstream) 301 self.check_setupdir_files(setuppath, test_file_content, config_upstream)
302 os.environ['BBPATH'] = os.path.join(buildpath, 'build') 302 os.environ['BBPATH'] = os.path.join(setuppath, 'build')
303 out = self.runbbsetup("status") 303 out = self.runbbsetup("status")
304 self.assertIn("Configuration in {} has not changed".format(buildpath), out[0]) 304 self.assertIn("Configuration in {} has not changed".format(setuppath), out[0])
305 out = self.runbbsetup("update") 305 out = self.runbbsetup("update")
306 self.assertIn("Configuration in {} has not changed".format(buildpath), out[0]) 306 self.assertIn("Configuration in {} has not changed".format(setuppath), out[0])
307 307
308 # install buildtools 308 # install buildtools
309 out = self.runbbsetup("install-buildtools") 309 out = self.runbbsetup("install-buildtools")
310 self.assertIn("Buildtools installed into", out[0]) 310 self.assertIn("Buildtools installed into", out[0])
311 self.assertTrue(os.path.exists(os.path.join(buildpath, 'buildtools'))) 311 self.assertTrue(os.path.exists(os.path.join(setuppath, 'buildtools')))
312 312
313 # change a file in the test layer repo, make a new commit and 313 # change a file in the test layer repo, make a new commit and
314 # test that status/update correctly report the change and update the config 314 # test that status/update correctly report the change and update the config
@@ -319,17 +319,17 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
319 'gizmo-env-passthrough', 319 'gizmo-env-passthrough',
320 'gizmo-no-fragment', 320 'gizmo-no-fragment',
321 'gadget-notemplate', 'gizmo-notemplate'): 321 'gadget-notemplate', 'gizmo-notemplate'):
322 buildpath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c)) 322 setuppath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c))
323 os.environ['BBPATH'] = os.path.join(buildpath, 'build') 323 os.environ['BBPATH'] = os.path.join(setuppath, 'build')
324 out = self.runbbsetup("status") 324 out = self.runbbsetup("status")
325 self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, buildpath), out[0]) 325 self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, setuppath), out[0])
326 out = self.runbbsetup("update") 326 out = self.runbbsetup("update")
327 if c in ('gadget', 'gizmo'): 327 if c in ('gadget', 'gizmo'):
328 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(buildpath), out[0]) 328 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(setuppath), out[0])
329 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0]) 329 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
330 with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f: 330 with open(os.path.join(setuppath, 'config', "config-upstream.json")) as f:
331 config_upstream = json.load(f) 331 config_upstream = json.load(f)
332 self.check_builddir_files(buildpath, test_file_content, config_upstream) 332 self.check_setupdir_files(setuppath, test_file_content, config_upstream)
333 333
334 # make a new branch in the test layer repo, change a file on that branch, 334 # make a new branch in the test layer repo, change a file on that branch,
335 # make a new commit, update the top level json config to refer to that branch, 335 # make a new commit, update the top level json config to refer to that branch,
@@ -344,15 +344,15 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
344 'gizmo-env-passthrough', 344 'gizmo-env-passthrough',
345 'gizmo-no-fragment', 345 'gizmo-no-fragment',
346 'gadget-notemplate', 'gizmo-notemplate'): 346 'gadget-notemplate', 'gizmo-notemplate'):
347 buildpath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c)) 347 setuppath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c))
348 os.environ['BBPATH'] = os.path.join(buildpath, 'build') 348 os.environ['BBPATH'] = os.path.join(setuppath, 'build')
349 out = self.runbbsetup("status") 349 out = self.runbbsetup("status")
350 self.assertIn("Configuration in {} has changed:".format(buildpath), out[0]) 350 self.assertIn("Configuration in {} has changed:".format(setuppath), out[0])
351 self.assertIn('- "rev": "master"\n+ "rev": "another-branch"', out[0]) 351 self.assertIn('- "rev": "master"\n+ "rev": "another-branch"', out[0])
352 out = self.runbbsetup("update") 352 out = self.runbbsetup("update")
353 if c in ('gadget', 'gizmo'): 353 if c in ('gadget', 'gizmo'):
354 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(buildpath), out[0]) 354 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(setuppath), out[0])
355 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0]) 355 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
356 with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f: 356 with open(os.path.join(setuppath, 'config', "config-upstream.json")) as f:
357 config_upstream = json.load(f) 357 config_upstream = json.load(f)
358 self.check_builddir_files(buildpath, test_file_content, config_upstream) 358 self.check_setupdir_files(setuppath, test_file_content, config_upstream)
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 577e765f11..37b0a75199 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -6,6 +6,11 @@
6# SPDX-License-Identifier: GPL-2.0-only 6# SPDX-License-Identifier: GPL-2.0-only
7# 7#
8 8
9# DEBUGGING NOTE: if you need to add debug statements into the files within a
10# Toaster-imported layer, you must disable the "git fetch ..." command after
11# the initial clone else any changes will be automatically replaced with the
12# original code
13
9import os 14import os
10import re 15import re
11import shutil 16import shutil
@@ -183,6 +188,8 @@ class LocalhostBEController(BuildEnvironmentController):
183 if commit != "HEAD": 188 if commit != "HEAD":
184 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) 189 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
185 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit 190 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
191 # DEBUGGING NOTE: this is the 'git fetch" to disable after the initial clone to
192 # prevent inserted debugging commands from being lost
186 self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env) 193 self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env)
187 194
188 # take the localdirname as poky dir if we can find the oe-init-build-env 195 # take the localdirname as poky dir if we can find the oe-init-build-env
@@ -319,7 +326,28 @@ class LocalhostBEController(BuildEnvironmentController):
319 layers = self.setLayers(bitbake, layers, targets) 326 layers = self.setLayers(bitbake, layers, targets)
320 is_merged_attr = bitbake.req.project.merged_attr 327 is_merged_attr = bitbake.req.project.merged_attr
321 328
329 # Is this build for a Bitbake Setup installation?
330 isBBSetup = False
331 release_description = bitbake.req.project.release.description
332 if 'master' in release_description:
333 # This release going forward is Bitbake Setup
334 isBBSetup = True
335 else:
336 try:
337 # Yocto Project 5.3 "Whinlatter"
338 params = release_description.split(' ')
339 # Work past YP "10.1"
340 release_params = params[2].split('.')
341 if int(release_params[0]) > 5:
342 isBBSetup = True
343 if (release_params[0] == '5') and (int(release_params[1]) >= 3):
344 isBBSetup = True
345 except:
346 # If release schema not recognized, then this release is in the future
347 isBBSetup = True
348
322 git_env = os.environ.copy() 349 git_env = os.environ.copy()
350 toaster_run_dir = git_env['BUILDDIR']
323 # (note: add custom environment settings here) 351 # (note: add custom environment settings here)
324 try: 352 try:
325 # insure that the project init/build uses the selected bitbake, and not Toaster's 353 # insure that the project init/build uses the selected bitbake, and not Toaster's
@@ -335,28 +363,44 @@ class LocalhostBEController(BuildEnvironmentController):
335 else: 363 else:
336 builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) 364 builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
337 oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') 365 oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
338 # init build environment 366 setup_init = os.path.join(builddir, 'init-build-env')
367 # init build environment and build directory structure
339 try: 368 try:
340 custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value 369 custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
341 custom_script = custom_script.replace("%BUILDDIR%" ,builddir) 370 custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
342 self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env) 371 self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
343 except ToasterSetting.DoesNotExist: 372 except ToasterSetting.DoesNotExist:
344 self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), 373 # Normal setup
345 self.be.sourcedir,env=git_env) 374 if isBBSetup:
375 # Create an 'init-build-env' for the build directory that points to respective cloned bitbake
376 os.makedirs(builddir, exist_ok = True)
377 build_init_script = os.path.join(builddir,'init-build-env')
378 os.system(f"cp {os.path.join(toaster_run_dir,'init-build-env')} {build_init_script}")
379 os.system(f"sed -i -e 's|^cd .*$|cd {self.pokydirname}|g' {build_init_script}")
380 os.system(f"sed -i -e 's|^set .*$|set {builddir}|g' {build_init_script}")
381 # Execute the init
382 self._shellcmd(f"bash -c 'source {build_init_script}'", env=git_env)
383 else:
384 self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
385 self.be.sourcedir,env=git_env)
346 386
347 # update bblayers.conf 387 # update bblayers.conf
348 if not is_merged_attr: 388 if not is_merged_attr:
349 bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") 389 bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
350 with open(bblconfpath, 'w') as bblayers: 390 with open(bblconfpath, 'w') as bblayers:
351 bblayers.write('# line added by toaster build control\n' 391 bblayers.write('# line added by toaster build control\n'
352 'BBLAYERS = "%s"' % ' '.join(layers)) 392 'BBLAYERS = "%s"\n' % ' '.join(layers))
353 393
354 # write configuration file 394 # write configuration file
355 confpath = os.path.join(builddir, 'conf/toaster.conf') 395 confpath = os.path.join(builddir, 'conf/toaster.conf')
356 with open(confpath, 'w') as conf: 396 with open(confpath, 'w') as conf:
397 if isBBSetup:
398 # Force TOPDIR
399 conf.write(f'# Force TOPDIR\n')
400 conf.write(f'TOPDIR = "{builddir}"\n\n')
357 for var in variables: 401 for var in variables:
358 conf.write('%s="%s"\n' % (var.name, var.value)) 402 conf.write('%s = "%s"\n' % (var.name, var.value))
359 conf.write('INHERIT+="toaster buildhistory"') 403 conf.write('INHERIT += "toaster buildhistory"\n')
360 else: 404 else:
361 # Append the Toaster-specific values directly to the bblayers.conf 405 # Append the Toaster-specific values directly to the bblayers.conf
362 bblconfpath = os.path.join(builddir, "conf/bblayers.conf") 406 bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
@@ -443,15 +487,25 @@ class LocalhostBEController(BuildEnvironmentController):
443 else: 487 else:
444 logger.error("Looks like Bitbake is not available, please fix your environment") 488 logger.error("Looks like Bitbake is not available, please fix your environment")
445 489
490 # NOTE: BB SETUP requires a 'bblayers.conf' even though Toaster immediately overrides it with 'toaster-bblayers.conf'
491 bblayers = os.path.join(builddir,"conf/bblayers.conf")
446 toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") 492 toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
447 if not is_merged_attr: 493 if isBBSetup:
448 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' 494 # Use 'init-build-env' model
449 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, 495 if not is_merged_attr:
450 builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) 496 self._shellcmd(f'{env_clean} bash -c \"source {setup_init}; BITBAKE_UI="knotty" {bitbake} --read {confpath} --read {bblayers} --read {toasterlayers} '
497 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
498 else:
499 self._shellcmd(f'{env_clean} bash -c \"source {setup_init}; BITBAKE_UI="knotty" {bitbake} '
500 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
451 else: 501 else:
452 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s ' 502 # Use 'oe-init-build-env' model
453 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, 503 if not is_merged_attr:
454 builddir, bitbake), self.be.sourcedir) 504 self._shellcmd(f'{env_clean} bash -c \"source {oe_init} {builddir}; BITBAKE_UI="knotty" {bitbake} --read {confpath} --read {bblayers} --read {toasterlayers} '
505 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
506 else:
507 self._shellcmd(f'{env_clean} bash -c \"source {oe_init} {builddir}; BITBAKE_UI="knotty" {bitbake} '
508 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
455 509
456 # read port number from bitbake.lock 510 # read port number from bitbake.lock
457 self.be.bbport = -1 511 self.be.bbport = -1
@@ -497,20 +551,17 @@ class LocalhostBEController(BuildEnvironmentController):
497 log = os.path.join(builddir, 'toaster_ui.log') 551 log = os.path.join(builddir, 'toaster_ui.log')
498 local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 552 local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
499 'bitbake') 553 'bitbake')
554
500 if not is_merged_attr: 555 if not is_merged_attr:
501 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' 556 self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
502 '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' 557 f'{bitbake} {bbtargets} -u toasterui --read {confpath} --read {bblayers} --read {toasterlayers} --token="" >>{log} 2>&1;'
503 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ 558 f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'],
504 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
505 self.be.bbport, bitbake,)],
506 builddir, nowait=True) 559 builddir, nowait=True)
507 else: 560 else:
508 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' 561 self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
509 '%s %s -u toasterui --token="" >>%s 2>&1;' 562 f'{local_bitbake} {bbtargets} -u toasterui --token="" >>{log} 2>&1;'
510 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ 563 f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'],
511 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
512 self.be.bbport, bitbake,)],
513 builddir, nowait=True) 564 builddir, nowait=True)
514 565
515 logger.debug('localhostbecontroller: Build launched, exiting. ' 566 logger.debug('localhostbecontroller: Build launched, exiting. '
516 'Follow build logs at %s' % log) 567 f'Follow build logs at {log}')