summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake')
-rwxr-xr-xbitbake/bin/bitbake-setup175
-rwxr-xr-xbitbake/bin/toaster21
-rw-r--r--bitbake/lib/bb/tests/setup.py88
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py99
4 files changed, 233 insertions, 150 deletions
diff --git a/bitbake/bin/bitbake-setup b/bitbake/bin/bitbake-setup
index a0426005ff..29fdf11892 100755
--- a/bitbake/bin/bitbake-setup
+++ b/bitbake/bin/bitbake-setup
@@ -18,6 +18,7 @@ import configparser
18import datetime 18import datetime
19import glob 19import glob
20import subprocess 20import subprocess
21import copy
21 22
22default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry") 23default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")
23 24
@@ -57,9 +58,15 @@ def get_config_name(config):
57 else: 58 else:
58 raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix)) 59 raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix))
59 60
60def write_config(config, config_dir): 61def write_upstream_config(config_dir, config_data):
61 with open(os.path.join(config_dir, "config-upstream.json"),'w') as s: 62 with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
62 json.dump(config, s, sort_keys=True, indent=4) 63 json.dump(config_data, s, sort_keys=True, indent=4)
64
65def write_sources_fixed_revisions(config_dir, config_data):
66 sources = {}
67 sources['sources'] = config_data
68 with open(os.path.join(config_dir, "sources-fixed-revisions.json"),'w') as s:
69 json.dump(sources, s, sort_keys=True, indent=4)
63 70
64def commit_config(config_dir): 71def commit_config(config_dir):
65 bb.process.run("git -C {} add .".format(config_dir)) 72 bb.process.run("git -C {} add .".format(config_dir))
@@ -76,6 +83,7 @@ def _write_layer_list(dest, repodirs):
76 json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4) 83 json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)
77 84
78def checkout_layers(layers, layerdir, d): 85def checkout_layers(layers, layerdir, d):
86 layers_fixed_revisions = copy.deepcopy(layers)
79 repodirs = [] 87 repodirs = []
80 oesetupbuild = None 88 oesetupbuild = None
81 print("Fetching layer/tool repositories into {}".format(layerdir)) 89 print("Fetching layer/tool repositories into {}".format(layerdir))
@@ -90,14 +98,18 @@ def checkout_layers(layers, layerdir, d):
90 remotes = r_remote['remotes'] 98 remotes = r_remote['remotes']
91 99
92 for remote in remotes: 100 for remote in remotes:
93 type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"]) 101 prot,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
94 fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) 102 fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
95 print(" {}".format(r_name)) 103 print(" {}".format(r_name))
96 if branch: 104 if branch:
97 fetcher = bb.fetch.Fetch(["{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir)], d) 105 src_uri = f"{fetchuri};protocol={prot};rev={rev};branch={branch};destsuffix={repodir}"
98 else: 106 else:
99 fetcher = bb.fetch.Fetch(["{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir)], d) 107 src_uri = f"{fetchuri};protocol={prot};rev={rev};nobranch=1;destsuffix={repodir}"
108 fetcher = bb.fetch.Fetch([src_uri], d)
100 do_fetch(fetcher, layerdir) 109 do_fetch(fetcher, layerdir)
110 urldata = fetcher.ud[src_uri]
111 revision = urldata.revision
112 layers_fixed_revisions[r_name]['git-remote']['rev'] = revision
101 113
102 if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')): 114 if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
103 oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build') 115 oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
@@ -114,22 +126,27 @@ def checkout_layers(layers, layerdir, d):
114 os.remove(symlink) 126 os.remove(symlink)
115 os.symlink(os.path.relpath(t,layerdir),symlink) 127 os.symlink(os.path.relpath(t,layerdir),symlink)
116 128
117def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir): 129 return layers_fixed_revisions
118 def _setup_build_conf(layers, build_conf_dir): 130
131def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir):
132 def _setup_build_conf(layers, relative_layers, build_conf_dir):
119 os.makedirs(build_conf_dir) 133 os.makedirs(build_conf_dir)
120 layers_s = [] 134 layers_s = []
135
121 for l in layers: 136 for l in layers:
122 if l.startswith("{THISDIR}/"): 137 l = os.path.join(layerdir, l)
123 if thisdir:
124 l = l.format(THISDIR=thisdir)
125 else:
126 raise Exception("Configuration is using {THISDIR} to specify " \
127 "a layer path relative to itself. This can be done only " \
128 "when the configuration is specified by its path on local " \
129 "disk, not when it's in a registry or is fetched over http.")
130 if not os.path.isabs(l):
131 l = os.path.join(layerdir, l)
132 layers_s.append(" {} \\".format(l)) 138 layers_s.append(" {} \\".format(l))
139
140 for l in relative_layers:
141 if thisdir:
142 l = os.path.join(thisdir, l)
143 else:
144 raise Exception("Configuration is using bb-layers-relative to specify " \
145 "a layer path relative to itself. This can be done only " \
146 "when the configuration is specified by its path on local " \
147 "disk, not when it's in a registry or is fetched over http.")
148 layers_s.append(" {} \\".format(l))
149
133 layers_s = "\n".join(layers_s) 150 layers_s = "\n".join(layers_s)
134 bblayers_conf = """BBLAYERS ?= " \\ 151 bblayers_conf = """BBLAYERS ?= " \\
135{} 152{}
@@ -188,7 +205,7 @@ def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
188 f.write('\n') 205 f.write('\n')
189 f.write(content) 206 f.write(content)
190 207
191 bitbake_builddir = os.path.join(builddir, "build") 208 bitbake_builddir = os.path.join(setupdir, "build")
192 print("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir)) 209 print("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir))
193 210
194 template = bitbake_config.get("oe-template") 211 template = bitbake_config.get("oe-template")
@@ -206,7 +223,8 @@ def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
206 os.rename(bitbake_confdir, backup_bitbake_confdir) 223 os.rename(bitbake_confdir, backup_bitbake_confdir)
207 224
208 if layers: 225 if layers:
209 _setup_build_conf(layers, bitbake_confdir) 226 relative_layers = bitbake_config.get("bb-layers-relative") or []
227 _setup_build_conf(layers, relative_layers, bitbake_confdir)
210 228
211 if template: 229 if template:
212 bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir)) 230 bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir))
@@ -220,7 +238,7 @@ def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
220 _prepend_passthrough_to_init_build_env(bitbake_builddir) 238 _prepend_passthrough_to_init_build_env(bitbake_builddir)
221 239
222 siteconf_symlink = os.path.join(bitbake_confdir, "site.conf") 240 siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
223 siteconf = os.path.normpath(os.path.join(builddir, '..', "site.conf")) 241 siteconf = os.path.normpath(os.path.join(setupdir, '..', "site.conf"))
224 if os.path.lexists(siteconf_symlink): 242 if os.path.lexists(siteconf_symlink):
225 os.remove(symlink) 243 os.remove(symlink)
226 os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink) 244 os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)
@@ -267,16 +285,17 @@ def get_registry_config(registry_path, id):
267 return os.path.join(root, f) 285 return os.path.join(root, f)
268 raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id)) 286 raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))
269 287
270def update_build(config, confdir, builddir, layerdir, d): 288def update_build(config, confdir, setupdir, layerdir, d):
271 layer_config = config["data"]["sources"] 289 layer_config = copy.deepcopy(config["data"]["sources"])
272 layer_overrides = config["source-overrides"]["sources"] 290 layer_overrides = config["source-overrides"]["sources"]
273 for k,v in layer_overrides.items(): 291 for k,v in layer_overrides.items():
274 if k in layer_config: 292 if k in layer_config:
275 layer_config[k]["git-remote"] = v["git-remote"] 293 layer_config[k]["git-remote"] = v["git-remote"]
276 checkout_layers(layer_config, layerdir, d) 294 sources_fixed_revisions = checkout_layers(layer_config, layerdir, d)
277 bitbake_config = config["bitbake-config"] 295 bitbake_config = config["bitbake-config"]
278 thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None 296 thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
279 setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir) 297 setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir)
298 write_sources_fixed_revisions(confdir, sources_fixed_revisions)
280 299
281def int_input(allowed_values): 300def int_input(allowed_values):
282 n = None 301 n = None
@@ -441,22 +460,22 @@ def init_config(top_dir, settings, args, d):
441 upstream_config = obtain_config(top_dir, settings, args, source_overrides, d) 460 upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
442 print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options']))) 461 print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))
443 462
444 builddir = os.path.join(os.path.abspath(top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_"))) 463 setupdir = os.path.join(os.path.abspath(top_dir), args.setup_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
445 if os.path.exists(os.path.join(builddir, "layers")): 464 if os.path.exists(os.path.join(setupdir, "layers")):
446 print(f"Build already initialized in:\n {builddir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize a build in this directory, remove it, and run 'bitbake-setup init' again.") 465 print(f"Setup already initialized in:\n {setupdir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize in this directory, remove it, and run 'bitbake-setup init' again.")
447 return 466 return
448 467
449 print("Initializing a build in\n {}".format(builddir)) 468 print("Initializing a setup directory in\n {}".format(setupdir))
450 if not args.non_interactive: 469 if not args.non_interactive:
451 y_or_n = input('Continue? (y/N): ') 470 y_or_n = input('Continue? (y/N): ')
452 if y_or_n != 'y': 471 if y_or_n != 'y':
453 exit() 472 exit()
454 print() 473 print()
455 474
456 os.makedirs(builddir, exist_ok=True) 475 os.makedirs(setupdir, exist_ok=True)
457 476
458 confdir = os.path.join(builddir, "config") 477 confdir = os.path.join(setupdir, "config")
459 layerdir = os.path.join(builddir, "layers") 478 layerdir = os.path.join(setupdir, "layers")
460 479
461 os.makedirs(confdir) 480 os.makedirs(confdir)
462 os.makedirs(layerdir) 481 os.makedirs(layerdir)
@@ -469,9 +488,9 @@ def init_config(top_dir, settings, args, d):
469 488
470 bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d) 489 bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)
471 490
472 write_config(upstream_config, confdir) 491 write_upstream_config(confdir, upstream_config)
492 update_build(upstream_config, confdir, setupdir, layerdir, d)
473 commit_config(confdir) 493 commit_config(confdir)
474 update_build(upstream_config, confdir, builddir, layerdir, d)
475 494
476 bb.event.remove("bb.build.TaskProgress", None) 495 bb.event.remove("bb.build.TaskProgress", None)
477 496
@@ -513,10 +532,10 @@ def are_layers_changed(layers, layerdir, d):
513 return changed 532 return changed
514 533
515def build_status(top_dir, settings, args, d, update=False): 534def build_status(top_dir, settings, args, d, update=False):
516 builddir = args.build_dir 535 setupdir = args.setup_dir
517 536
518 confdir = os.path.join(builddir, "config") 537 confdir = os.path.join(setupdir, "config")
519 layerdir = os.path.join(builddir, "layers") 538 layerdir = os.path.join(setupdir, "layers")
520 539
521 current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json"))) 540 current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))
522 541
@@ -526,24 +545,24 @@ def build_status(top_dir, settings, args, d, update=False):
526 source_overrides = current_upstream_config["source-overrides"] 545 source_overrides = current_upstream_config["source-overrides"]
527 new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d) 546 new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
528 547
529 write_config(new_upstream_config, confdir) 548 write_upstream_config(confdir, new_upstream_config)
530 config_diff = bb.process.run('git -C {} diff'.format(confdir))[0] 549 config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
531 550
532 if config_diff: 551 if config_diff:
533 print('\nConfiguration in {} has changed:\n{}'.format(builddir, config_diff)) 552 print('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff))
534 if update: 553 if update:
535 commit_config(confdir) 554 commit_config(confdir)
536 update_build(new_upstream_config, confdir, builddir, layerdir, d) 555 update_build(new_upstream_config, confdir, setupdir, layerdir, d)
537 else: 556 else:
538 bb.process.run('git -C {} restore config-upstream.json'.format(confdir)) 557 bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
539 return 558 return
540 559
541 if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d): 560 if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
542 if update: 561 if update:
543 update_build(current_upstream_config, confdir, builddir, layerdir, d) 562 update_build(current_upstream_config, confdir, setupdir, layerdir, d)
544 return 563 return
545 564
546 print("\nConfiguration in {} has not changed.".format(builddir)) 565 print("\nConfiguration in {} has not changed.".format(setupdir))
547 566
548def build_update(top_dir, settings, args, d): 567def build_update(top_dir, settings, args, d):
549 build_status(top_dir, settings, args, d, update=True) 568 build_status(top_dir, settings, args, d, update=True)
@@ -616,7 +635,7 @@ def list_configs(top_dir, settings, args, d):
616 print("Available configurations written into {}".format(args.write_json)) 635 print("Available configurations written into {}".format(args.write_json))
617 636
618def install_buildtools(top_dir, settings, args, d): 637def install_buildtools(top_dir, settings, args, d):
619 buildtools_install_dir = os.path.join(args.build_dir, 'buildtools') 638 buildtools_install_dir = os.path.join(args.setup_dir, 'buildtools')
620 if os.path.exists(buildtools_install_dir): 639 if os.path.exists(buildtools_install_dir):
621 if not args.force: 640 if not args.force:
622 print("Buildtools are already installed in {}.".format(buildtools_install_dir)) 641 print("Buildtools are already installed in {}.".format(buildtools_install_dir))
@@ -629,14 +648,11 @@ def install_buildtools(top_dir, settings, args, d):
629 return 648 return
630 shutil.rmtree(buildtools_install_dir) 649 shutil.rmtree(buildtools_install_dir)
631 650
632 install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools') 651 install_buildtools = os.path.join(args.setup_dir, 'layers/oe-scripts/install-buildtools')
633 buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S"))) 652 buildtools_download_dir = os.path.join(args.setup_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
634 print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir)) 653 print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
635 subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True) 654 subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)
636 655
637def default_settings_path(top_dir):
638 return os.path.join(top_dir, 'settings.conf')
639
640def create_siteconf(top_dir, non_interactive=True): 656def create_siteconf(top_dir, non_interactive=True):
641 siteconfpath = os.path.join(top_dir, 'site.conf') 657 siteconfpath = os.path.join(top_dir, 'site.conf')
642 print('A common site.conf file will be created, please edit or replace before running builds\n {}\n'.format(siteconfpath)) 658 print('A common site.conf file will be created, please edit or replace before running builds\n {}\n'.format(siteconfpath))
@@ -653,6 +669,9 @@ def create_siteconf(top_dir, non_interactive=True):
653 with open(siteconfpath, 'w') as siteconffile: 669 with open(siteconfpath, 'w') as siteconffile:
654 siteconffile.write('# This file is intended for build host-specific bitbake settings\n') 670 siteconffile.write('# This file is intended for build host-specific bitbake settings\n')
655 671
672def topdir_settings_path(top_dir):
673 return os.path.join(top_dir, 'settings.conf')
674
656def global_settings_path(args): 675def global_settings_path(args):
657 return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf') 676 return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf')
658 677
@@ -667,7 +686,7 @@ def change_setting(top_dir, args):
667 if vars(args)['global']: 686 if vars(args)['global']:
668 settings_path = global_settings_path(args) 687 settings_path = global_settings_path(args)
669 else: 688 else:
670 settings_path = default_settings_path(top_dir) 689 settings_path = topdir_settings_path(top_dir)
671 settings = load_settings(settings_path) 690 settings = load_settings(settings_path)
672 691
673 if args.subcommand == 'set': 692 if args.subcommand == 'set':
@@ -696,50 +715,54 @@ def settings_func(top_dir, all_settings, args):
696 elif args.subcommand == 'set' or args.subcommand == 'unset': 715 elif args.subcommand == 'set' or args.subcommand == 'unset':
697 change_setting(top_dir, args) 716 change_setting(top_dir, args)
698 717
699def get_build_dir_via_bbpath(): 718def get_setup_dir_via_bbpath():
700 bbpath = os.environ.get('BBPATH') 719 bbpath = os.environ.get('BBPATH')
701 if bbpath: 720 if bbpath:
702 bitbake_dir = os.path.normpath(bbpath.split(':')[0]) 721 bitbake_dir = os.path.normpath(bbpath.split(':')[0])
703 if os.path.exists(os.path.join(bitbake_dir,'init-build-env')): 722 if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
704 build_dir = os.path.dirname(bitbake_dir) 723 setup_dir = os.path.dirname(bitbake_dir)
705 return build_dir 724 return setup_dir
706 return None 725 return None
707 726
708def get_top_dir(args, settings): 727def get_top_dir(args, settings):
709 build_dir_via_bbpath = get_build_dir_via_bbpath() 728 setup_dir_via_bbpath = get_setup_dir_via_bbpath()
710 if build_dir_via_bbpath: 729 if setup_dir_via_bbpath:
711 top_dir = os.path.dirname(build_dir_via_bbpath) 730 top_dir = os.path.dirname(setup_dir_via_bbpath)
712 if os.path.exists(default_settings_path(top_dir)): 731 if os.path.exists(topdir_settings_path(top_dir)):
713 return top_dir 732 return top_dir
714 733
715 if hasattr(args, 'build_dir'): 734 if hasattr(args, 'setup_dir'):
716 top_dir = os.path.dirname(os.path.normpath(args.build_dir)) 735 top_dir = os.path.dirname(os.path.normpath(args.setup_dir))
717 return top_dir 736 return top_dir
718 737
719 top_dir_prefix = settings['default']['top-dir-prefix'] 738 top_dir_prefix = settings['default']['top-dir-prefix']
720 top_dir_name = settings['default']['top-dir-name'] 739 top_dir_name = settings['default']['top-dir-name']
721 return os.path.join(top_dir_prefix, top_dir_name) 740 return os.path.join(top_dir_prefix, top_dir_name)
722 741
723def merge_settings(builtin_settings, global_settings, local_settings, cmdline_settings): 742def merge_settings(builtin_settings, global_settings, topdir_settings, cmdline_settings):
724 all_settings = builtin_settings 743 all_settings = builtin_settings
725 744
726 for s in (global_settings, local_settings): 745 for s in (global_settings, topdir_settings):
727 for section, section_settings in s.items(): 746 for section, section_settings in s.items():
728 for setting, value in section_settings.items(): 747 for setting, value in section_settings.items():
748 if section not in all_settings.keys():
749 all_settings[section] = {}
729 all_settings[section][setting] = value 750 all_settings[section][setting] = value
730 751
731 for (section, setting, value) in cmdline_settings: 752 for (section, setting, value) in cmdline_settings:
753 if section not in all_settings.keys():
754 all_settings[section] = {}
732 all_settings[section][setting] = value 755 all_settings[section][setting] = value
733 756
734 return all_settings 757 return all_settings
735 758
736def main(): 759def main():
737 def add_build_dir_arg(parser): 760 def add_setup_dir_arg(parser):
738 build_dir = get_build_dir_via_bbpath() 761 setup_dir = get_setup_dir_via_bbpath()
739 if build_dir: 762 if setup_dir:
740 parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH") 763 parser.add_argument('--setup-dir', default=setup_dir, help="Path to the setup, default is %(default)s via BBPATH")
741 else: 764 else:
742 parser.add_argument('--build-dir', required=True, help="Path to the build") 765 parser.add_argument('--setup-dir', required=True, help="Path to the setup")
743 766
744 parser = argparse.ArgumentParser( 767 parser = argparse.ArgumentParser(
745 description="BitBake setup utility. Run with 'init' argument to get started.", 768 description="BitBake setup utility. Run with 'init' argument to get started.",
@@ -761,24 +784,24 @@ def main():
761 parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.') 784 parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
762 parser_list.set_defaults(func=list_configs) 785 parser_list.set_defaults(func=list_configs)
763 786
764 parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it') 787 parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a setup from it')
765 parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.") 788 parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
766 parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.') 789 parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
767 parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.') 790 parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
768 parser_init.add_argument('--build-dir-name', action='store', help='A custom build directory name under the top directory.') 791 parser_init.add_argument('--setup-dir-name', action='store', help='A custom setup directory name under the top directory.')
769 parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.') 792 parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
770 parser_init.set_defaults(func=init_config) 793 parser_init.set_defaults(func=init_config)
771 794
772 parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration') 795 parser_status = subparsers.add_parser('status', help='Check if the setup needs to be synchronized with configuration')
773 add_build_dir_arg(parser_status) 796 add_setup_dir_arg(parser_status)
774 parser_status.set_defaults(func=build_status) 797 parser_status.set_defaults(func=build_status)
775 798
776 parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration') 799 parser_update = subparsers.add_parser('update', help='Update a setup to be in sync with configuration')
777 add_build_dir_arg(parser_update) 800 add_setup_dir_arg(parser_update)
778 parser_update.set_defaults(func=build_update) 801 parser_update.set_defaults(func=build_update)
779 802
780 parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine') 803 parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
781 add_build_dir_arg(parser_install_buildtools) 804 add_setup_dir_arg(parser_install_buildtools)
782 parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.') 805 parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
783 parser_install_buildtools.set_defaults(func=install_buildtools) 806 parser_install_buildtools.set_defaults(func=install_buildtools)
784 807
@@ -786,7 +809,7 @@ def main():
786 parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory") 809 parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory")
787 810
788 parser_settings = subparsers.add_parser('settings', 811 parser_settings = subparsers.add_parser('settings',
789 help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of build configuration registry, downloads directory and other settings specific to a top directory)') 812 help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of configuration registry, downloads directory and other settings specific to a top directory)')
790 parser_settings.set_defaults(func=settings_func) 813 parser_settings.set_defaults(func=settings_func)
791 814
792 subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file") 815 subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file")
@@ -820,9 +843,9 @@ def main():
820 level=logger.getEffectiveLevel()) 843 level=logger.getEffectiveLevel())
821 844
822 if 'func' in args: 845 if 'func' in args:
823 if hasattr(args, 'build_dir'): 846 if hasattr(args, 'setup_dir'):
824 if not os.path.exists(os.path.join(args.build_dir,'build', 'init-build-env')): 847 if not os.path.exists(os.path.join(args.setup_dir,'build', 'init-build-env')):
825 print("Not a valid build directory: build/init-build-env does not exist in {}".format(args.build_dir)) 848 print("Not a valid setup directory: build/init-build-env does not exist in {}".format(args.setup_dir))
826 return 849 return
827 850
828 if not hasattr(args, 'non_interactive'): 851 if not hasattr(args, 'non_interactive'):
@@ -841,7 +864,7 @@ def main():
841 # This cannot be set with the rest of the builtin settings as top_dir needs to be determined first 864 # This cannot be set with the rest of the builtin settings as top_dir needs to be determined first
842 builtin_settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads') 865 builtin_settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads')
843 866
844 topdir_settings = load_settings(default_settings_path(top_dir)) 867 topdir_settings = load_settings(topdir_settings_path(top_dir))
845 all_settings = merge_settings(builtin_settings, global_settings, topdir_settings, args.cmdline_settings) 868 all_settings = merge_settings(builtin_settings, global_settings, topdir_settings, args.cmdline_settings)
846 869
847 if args.func == settings_func: 870 if args.func == settings_func:
diff --git a/bitbake/bin/toaster b/bitbake/bin/toaster
index f002c8c159..d20cbe90ba 100755
--- a/bitbake/bin/toaster
+++ b/bitbake/bin/toaster
@@ -164,14 +164,20 @@ fi
164# in the local layers that currently make using an arbitrary 164# in the local layers that currently make using an arbitrary
165# toasterconf.json difficult. 165# toasterconf.json difficult.
166 166
167. $OE_ROOT/.templateconf 167# BBSETUP no longer supports .templateconf
168if [ -n "$TEMPLATECONF" ]; then 168if [ -f $OE_ROOT/.templateconf ] ; then
169 if [ ! -d "$TEMPLATECONF" ]; then 169 . $OE_ROOT/.templateconf
170 # Allow TEMPLATECONF=meta-xyz/conf as a shortcut 170 if [ -n "$TEMPLATECONF" ]; then
171 if [ -d "$OE_ROOT/$TEMPLATECONF" ]; then 171 if [ ! -d "$TEMPLATECONF" ]; then
172 TEMPLATECONF="$OE_ROOT/$TEMPLATECONF" 172 # Allow TEMPLATECONF=meta-xyz/conf as a shortcut
173 if [ -d "$OE_ROOT/$TEMPLATECONF" ]; then
174 TEMPLATECONF="$OE_ROOT/$TEMPLATECONF"
175 fi
173 fi 176 fi
174 fi 177 fi
178else
179 # Force poky
180 TEMPLATECONF="layers/meta-yocto/meta-poky/conf/templates/default"
175fi 181fi
176 182
177unset OE_ROOT 183unset OE_ROOT
@@ -255,7 +261,8 @@ fi
255# 2) the build dir (in build) 261# 2) the build dir (in build)
256# 3) the sqlite db if that is being used. 262# 3) the sqlite db if that is being used.
257# 4) pid's we need to clean up on exit/shutdown 263# 4) pid's we need to clean up on exit/shutdown
258export TOASTER_DIR=$TOASTERDIR 264# By default we move this all into the Toaster's parent build directory
265export TOASTER_DIR=$BUILDDIR
259export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS TOASTER_DIR" 266export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS TOASTER_DIR"
260 267
261# Determine the action. If specified by arguments, fine, if not, toggle it 268# Determine the action. If specified by arguments, fine, if not, toggle it
diff --git a/bitbake/lib/bb/tests/setup.py b/bitbake/lib/bb/tests/setup.py
index e320cdf56f..ba2a90009d 100644
--- a/bitbake/lib/bb/tests/setup.py
+++ b/bitbake/lib/bb/tests/setup.py
@@ -109,19 +109,19 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
109 "configurations": [ 109 "configurations": [
110 { 110 {
111 "name": "gadget", 111 "name": "gadget",
112 "description": "Gadget build configuration", 112 "description": "Gadget configuration",
113 "oe-template": "test-configuration-gadget", 113 "oe-template": "test-configuration-gadget",
114 "oe-fragments": ["test-fragment-1"] 114 "oe-fragments": ["test-fragment-1"]
115 }, 115 },
116 { 116 {
117 "name": "gizmo", 117 "name": "gizmo",
118 "description": "Gizmo build configuration", 118 "description": "Gizmo configuration",
119 "oe-template": "test-configuration-gizmo", 119 "oe-template": "test-configuration-gizmo",
120 "oe-fragments": ["test-fragment-2"] 120 "oe-fragments": ["test-fragment-2"]
121 }, 121 },
122 { 122 {
123 "name": "gizmo-env-passthrough", 123 "name": "gizmo-env-passthrough",
124 "description": "Gizmo build configuration with environment-passthrough", 124 "description": "Gizmo configuration with environment-passthrough",
125 "bb-layers": ["layerC","layerD/meta-layer"], 125 "bb-layers": ["layerC","layerD/meta-layer"],
126 "oe-fragments": ["test-fragment-1"], 126 "oe-fragments": ["test-fragment-1"],
127 "bb-env-passthrough-additions": [ 127 "bb-env-passthrough-additions": [
@@ -132,25 +132,26 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
132 }, 132 },
133 { 133 {
134 "name": "gizmo-no-fragment", 134 "name": "gizmo-no-fragment",
135 "description": "Gizmo no-fragment template-only build configuration", 135 "description": "Gizmo no-fragment template-only configuration",
136 "oe-template": "test-configuration-gizmo" 136 "oe-template": "test-configuration-gizmo"
137 }, 137 },
138 { 138 {
139 "name": "gadget-notemplate", 139 "name": "gadget-notemplate",
140 "description": "Gadget notemplate build configuration", 140 "description": "Gadget notemplate configuration",
141 "bb-layers": ["layerA","layerB/meta-layer"], 141 "bb-layers": ["layerA","layerB/meta-layer"],
142 "oe-fragments": ["test-fragment-1"] 142 "oe-fragments": ["test-fragment-1"]
143 }, 143 },
144 { 144 {
145 "name": "gizmo-notemplate", 145 "name": "gizmo-notemplate",
146 "description": "Gizmo notemplate build configuration", 146 "description": "Gizmo notemplate configuration",
147 "bb-layers": ["layerC","layerD/meta-layer"], 147 "bb-layers": ["layerC","layerD/meta-layer"],
148 "oe-fragments": ["test-fragment-2"] 148 "oe-fragments": ["test-fragment-2"]
149 }, 149 },
150 { 150 {
151 "name": "gizmo-notemplate-with-thisdir", 151 "name": "gizmo-notemplate-with-relative-layers",
152 "description": "Gizmo notemplate build configuration using THISDIR", 152 "description": "Gizmo notemplate configuration using relative layers",
153 "bb-layers": ["layerC","layerD/meta-layer","{THISDIR}/layerE/meta-layer"], 153 "bb-layers": ["layerC","layerD/meta-layer"],
154 "bb-layers-relative": ["layerE/meta-layer"],
154 "oe-fragments": ["test-fragment-2"] 155 "oe-fragments": ["test-fragment-2"]
155 } 156 }
156 ] 157 ]
@@ -177,14 +178,22 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
177 self.git('add {}'.format(name), cwd=self.testrepopath) 178 self.git('add {}'.format(name), cwd=self.testrepopath)
178 self.git('commit -m "Adding {}"'.format(name), cwd=self.testrepopath) 179 self.git('commit -m "Adding {}"'.format(name), cwd=self.testrepopath)
179 180
180 def check_builddir_files(self, buildpath, test_file_content, json_config): 181 def check_setupdir_files(self, setuppath, test_file_content):
181 with open(os.path.join(buildpath, 'layers', 'test-repo', 'test-file')) as f: 182 with open(os.path.join(setuppath, 'config', "config-upstream.json")) as f:
183 config_upstream = json.load(f)
184 with open(os.path.join(setuppath, 'layers', 'test-repo', 'test-file')) as f:
182 self.assertEqual(f.read(), test_file_content) 185 self.assertEqual(f.read(), test_file_content)
183 bitbake_config = json_config["bitbake-config"] 186 bitbake_config = config_upstream["bitbake-config"]
184 bb_build_path = os.path.join(buildpath, 'build') 187 bb_build_path = os.path.join(setuppath, 'build')
185 bb_conf_path = os.path.join(bb_build_path, 'conf') 188 bb_conf_path = os.path.join(bb_build_path, 'conf')
186 self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env'))) 189 self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env')))
187 190
191 with open(os.path.join(setuppath, 'config', "sources-fixed-revisions.json")) as f:
192 sources_fixed_revisions = json.load(f)
193 self.assertTrue('test-repo' in sources_fixed_revisions['sources'].keys())
194 revision = self.git('rev-parse HEAD', cwd=self.testrepopath).strip()
195 self.assertEqual(revision, sources_fixed_revisions['sources']['test-repo']['git-remote']['rev'])
196
188 if "oe-template" in bitbake_config: 197 if "oe-template" in bitbake_config:
189 with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f: 198 with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f:
190 self.assertEqual(f.read(), bitbake_config["oe-template"]) 199 self.assertEqual(f.read(), bitbake_config["oe-template"])
@@ -196,14 +205,13 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
196 with open(os.path.join(bb_conf_path, 'bblayers.conf')) as f: 205 with open(os.path.join(bb_conf_path, 'bblayers.conf')) as f:
197 bblayers = f.read() 206 bblayers = f.read()
198 for l in bitbake_config["bb-layers"]: 207 for l in bitbake_config["bb-layers"]:
199 if l.startswith('{THISDIR}/'): 208 self.assertIn(os.path.join(setuppath, "layers", l), bblayers)
200 thisdir_layer = os.path.join( 209 for l in bitbake_config.get("bb-layers-relative") or []:
201 os.path.dirname(json_config["path"]), 210 relative_layer = os.path.join(
202 l.removeprefix("{THISDIR}/"), 211 os.path.dirname(config_upstream["path"]),
212 l,
203 ) 213 )
204 self.assertIn(thisdir_layer, bblayers) 214 self.assertIn(relative_layer, bblayers)
205 else:
206 self.assertIn(os.path.join(buildpath, "layers", l), bblayers)
207 215
208 if 'oe-fragment' in bitbake_config.keys(): 216 if 'oe-fragment' in bitbake_config.keys():
209 for f in bitbake_config["oe-fragments"]: 217 for f in bitbake_config["oe-fragments"]:
@@ -290,25 +298,23 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
290 'gizmo-env-passthrough', 298 'gizmo-env-passthrough',
291 'gizmo-no-fragment', 299 'gizmo-no-fragment',
292 'gadget-notemplate','gizmo-notemplate', 300 'gadget-notemplate','gizmo-notemplate',
293 'gizmo-notemplate-with-thisdir')} 301 'gizmo-notemplate-with-relative-layers')}
294 } 302 }
295 for cf, v in test_configurations.items(): 303 for cf, v in test_configurations.items():
296 for c in v['buildconfigs']: 304 for c in v['buildconfigs']:
297 out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c)) 305 out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c))
298 buildpath = os.path.join(self.tempdir, 'bitbake-builds', '{}-{}'.format(cf, c)) 306 setuppath = os.path.join(self.tempdir, 'bitbake-builds', '{}-{}'.format(cf, c))
299 with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f: 307 self.check_setupdir_files(setuppath, test_file_content)
300 config_upstream = json.load(f) 308 os.environ['BBPATH'] = os.path.join(setuppath, 'build')
301 self.check_builddir_files(buildpath, test_file_content, config_upstream)
302 os.environ['BBPATH'] = os.path.join(buildpath, 'build')
303 out = self.runbbsetup("status") 309 out = self.runbbsetup("status")
304 self.assertIn("Configuration in {} has not changed".format(buildpath), out[0]) 310 self.assertIn("Configuration in {} has not changed".format(setuppath), out[0])
305 out = self.runbbsetup("update") 311 out = self.runbbsetup("update")
306 self.assertIn("Configuration in {} has not changed".format(buildpath), out[0]) 312 self.assertIn("Configuration in {} has not changed".format(setuppath), out[0])
307 313
308 # install buildtools 314 # install buildtools
309 out = self.runbbsetup("install-buildtools") 315 out = self.runbbsetup("install-buildtools")
310 self.assertIn("Buildtools installed into", out[0]) 316 self.assertIn("Buildtools installed into", out[0])
311 self.assertTrue(os.path.exists(os.path.join(buildpath, 'buildtools'))) 317 self.assertTrue(os.path.exists(os.path.join(setuppath, 'buildtools')))
312 318
313 # change a file in the test layer repo, make a new commit and 319 # change a file in the test layer repo, make a new commit and
314 # test that status/update correctly report the change and update the config 320 # test that status/update correctly report the change and update the config
@@ -319,17 +325,15 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
319 'gizmo-env-passthrough', 325 'gizmo-env-passthrough',
320 'gizmo-no-fragment', 326 'gizmo-no-fragment',
321 'gadget-notemplate', 'gizmo-notemplate'): 327 'gadget-notemplate', 'gizmo-notemplate'):
322 buildpath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c)) 328 setuppath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c))
323 os.environ['BBPATH'] = os.path.join(buildpath, 'build') 329 os.environ['BBPATH'] = os.path.join(setuppath, 'build')
324 out = self.runbbsetup("status") 330 out = self.runbbsetup("status")
325 self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, buildpath), out[0]) 331 self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, setuppath), out[0])
326 out = self.runbbsetup("update") 332 out = self.runbbsetup("update")
327 if c in ('gadget', 'gizmo'): 333 if c in ('gadget', 'gizmo'):
328 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(buildpath), out[0]) 334 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(setuppath), out[0])
329 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0]) 335 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
330 with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f: 336 self.check_setupdir_files(setuppath, test_file_content)
331 config_upstream = json.load(f)
332 self.check_builddir_files(buildpath, test_file_content, config_upstream)
333 337
334 # make a new branch in the test layer repo, change a file on that branch, 338 # make a new branch in the test layer repo, change a file on that branch,
335 # make a new commit, update the top level json config to refer to that branch, 339 # make a new commit, update the top level json config to refer to that branch,
@@ -344,15 +348,13 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
344 'gizmo-env-passthrough', 348 'gizmo-env-passthrough',
345 'gizmo-no-fragment', 349 'gizmo-no-fragment',
346 'gadget-notemplate', 'gizmo-notemplate'): 350 'gadget-notemplate', 'gizmo-notemplate'):
347 buildpath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c)) 351 setuppath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c))
348 os.environ['BBPATH'] = os.path.join(buildpath, 'build') 352 os.environ['BBPATH'] = os.path.join(setuppath, 'build')
349 out = self.runbbsetup("status") 353 out = self.runbbsetup("status")
350 self.assertIn("Configuration in {} has changed:".format(buildpath), out[0]) 354 self.assertIn("Configuration in {} has changed:".format(setuppath), out[0])
351 self.assertIn('- "rev": "master"\n+ "rev": "another-branch"', out[0]) 355 self.assertIn('- "rev": "master"\n+ "rev": "another-branch"', out[0])
352 out = self.runbbsetup("update") 356 out = self.runbbsetup("update")
353 if c in ('gadget', 'gizmo'): 357 if c in ('gadget', 'gizmo'):
354 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(buildpath), out[0]) 358 self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(setuppath), out[0])
355 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0]) 359 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
356 with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f: 360 self.check_setupdir_files(setuppath, test_file_content)
357 config_upstream = json.load(f)
358 self.check_builddir_files(buildpath, test_file_content, config_upstream)
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 577e765f11..37b0a75199 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -6,6 +6,11 @@
6# SPDX-License-Identifier: GPL-2.0-only 6# SPDX-License-Identifier: GPL-2.0-only
7# 7#
8 8
9# DEBUGGING NOTE: if you need to add debug statements into the files within a
10# Toaster-imported layer, you must disable the "git fetch ..." command after
11# the initial clone else any changes will be automatically replaced with the
12# original code
13
9import os 14import os
10import re 15import re
11import shutil 16import shutil
@@ -183,6 +188,8 @@ class LocalhostBEController(BuildEnvironmentController):
183 if commit != "HEAD": 188 if commit != "HEAD":
184 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) 189 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
185 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit 190 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
191 # DEBUGGING NOTE: this is the 'git fetch" to disable after the initial clone to
192 # prevent inserted debugging commands from being lost
186 self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env) 193 self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env)
187 194
188 # take the localdirname as poky dir if we can find the oe-init-build-env 195 # take the localdirname as poky dir if we can find the oe-init-build-env
@@ -319,7 +326,28 @@ class LocalhostBEController(BuildEnvironmentController):
319 layers = self.setLayers(bitbake, layers, targets) 326 layers = self.setLayers(bitbake, layers, targets)
320 is_merged_attr = bitbake.req.project.merged_attr 327 is_merged_attr = bitbake.req.project.merged_attr
321 328
329 # Is this build for a Bitbake Setup installation?
330 isBBSetup = False
331 release_description = bitbake.req.project.release.description
332 if 'master' in release_description:
333 # This release going forward is Bitbake Setup
334 isBBSetup = True
335 else:
336 try:
337 # Yocto Project 5.3 "Whinlatter"
338 params = release_description.split(' ')
339 # Work past YP "10.1"
340 release_params = params[2].split('.')
341 if int(release_params[0]) > 5:
342 isBBSetup = True
343 if (release_params[0] == '5') and (int(release_params[1]) >= 3):
344 isBBSetup = True
345 except:
346 # If release schema not recognized, then this release is in the future
347 isBBSetup = True
348
322 git_env = os.environ.copy() 349 git_env = os.environ.copy()
350 toaster_run_dir = git_env['BUILDDIR']
323 # (note: add custom environment settings here) 351 # (note: add custom environment settings here)
324 try: 352 try:
325 # insure that the project init/build uses the selected bitbake, and not Toaster's 353 # insure that the project init/build uses the selected bitbake, and not Toaster's
@@ -335,28 +363,44 @@ class LocalhostBEController(BuildEnvironmentController):
335 else: 363 else:
336 builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) 364 builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
337 oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') 365 oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
338 # init build environment 366 setup_init = os.path.join(builddir, 'init-build-env')
367 # init build environment and build directory structure
339 try: 368 try:
340 custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value 369 custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
341 custom_script = custom_script.replace("%BUILDDIR%" ,builddir) 370 custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
342 self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env) 371 self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
343 except ToasterSetting.DoesNotExist: 372 except ToasterSetting.DoesNotExist:
344 self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), 373 # Normal setup
345 self.be.sourcedir,env=git_env) 374 if isBBSetup:
375 # Create an 'init-build-env' for the build directory that points to respective cloned bitbake
376 os.makedirs(builddir, exist_ok = True)
377 build_init_script = os.path.join(builddir,'init-build-env')
378 os.system(f"cp {os.path.join(toaster_run_dir,'init-build-env')} {build_init_script}")
379 os.system(f"sed -i -e 's|^cd .*$|cd {self.pokydirname}|g' {build_init_script}")
380 os.system(f"sed -i -e 's|^set .*$|set {builddir}|g' {build_init_script}")
381 # Execute the init
382 self._shellcmd(f"bash -c 'source {build_init_script}'", env=git_env)
383 else:
384 self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
385 self.be.sourcedir,env=git_env)
346 386
347 # update bblayers.conf 387 # update bblayers.conf
348 if not is_merged_attr: 388 if not is_merged_attr:
349 bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") 389 bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
350 with open(bblconfpath, 'w') as bblayers: 390 with open(bblconfpath, 'w') as bblayers:
351 bblayers.write('# line added by toaster build control\n' 391 bblayers.write('# line added by toaster build control\n'
352 'BBLAYERS = "%s"' % ' '.join(layers)) 392 'BBLAYERS = "%s"\n' % ' '.join(layers))
353 393
354 # write configuration file 394 # write configuration file
355 confpath = os.path.join(builddir, 'conf/toaster.conf') 395 confpath = os.path.join(builddir, 'conf/toaster.conf')
356 with open(confpath, 'w') as conf: 396 with open(confpath, 'w') as conf:
397 if isBBSetup:
398 # Force TOPDIR
399 conf.write(f'# Force TOPDIR\n')
400 conf.write(f'TOPDIR = "{builddir}"\n\n')
357 for var in variables: 401 for var in variables:
358 conf.write('%s="%s"\n' % (var.name, var.value)) 402 conf.write('%s = "%s"\n' % (var.name, var.value))
359 conf.write('INHERIT+="toaster buildhistory"') 403 conf.write('INHERIT += "toaster buildhistory"\n')
360 else: 404 else:
361 # Append the Toaster-specific values directly to the bblayers.conf 405 # Append the Toaster-specific values directly to the bblayers.conf
362 bblconfpath = os.path.join(builddir, "conf/bblayers.conf") 406 bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
@@ -443,15 +487,25 @@ class LocalhostBEController(BuildEnvironmentController):
443 else: 487 else:
444 logger.error("Looks like Bitbake is not available, please fix your environment") 488 logger.error("Looks like Bitbake is not available, please fix your environment")
445 489
490 # NOTE: BB SETUP requires a 'bblayers.conf' even though Toaster immediately overrides it with 'toaster-bblayers.conf'
491 bblayers = os.path.join(builddir,"conf/bblayers.conf")
446 toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") 492 toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
447 if not is_merged_attr: 493 if isBBSetup:
448 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' 494 # Use 'init-build-env' model
449 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, 495 if not is_merged_attr:
450 builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) 496 self._shellcmd(f'{env_clean} bash -c \"source {setup_init}; BITBAKE_UI="knotty" {bitbake} --read {confpath} --read {bblayers} --read {toasterlayers} '
497 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
498 else:
499 self._shellcmd(f'{env_clean} bash -c \"source {setup_init}; BITBAKE_UI="knotty" {bitbake} '
500 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
451 else: 501 else:
452 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s ' 502 # Use 'oe-init-build-env' model
453 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, 503 if not is_merged_attr:
454 builddir, bitbake), self.be.sourcedir) 504 self._shellcmd(f'{env_clean} bash -c \"source {oe_init} {builddir}; BITBAKE_UI="knotty" {bitbake} --read {confpath} --read {bblayers} --read {toasterlayers} '
505 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
506 else:
507 self._shellcmd(f'{env_clean} bash -c \"source {oe_init} {builddir}; BITBAKE_UI="knotty" {bitbake} '
508 '--server-only -B 0.0.0.0:0\"', self.be.sourcedir)
455 509
456 # read port number from bitbake.lock 510 # read port number from bitbake.lock
457 self.be.bbport = -1 511 self.be.bbport = -1
@@ -497,20 +551,17 @@ class LocalhostBEController(BuildEnvironmentController):
497 log = os.path.join(builddir, 'toaster_ui.log') 551 log = os.path.join(builddir, 'toaster_ui.log')
498 local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 552 local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
499 'bitbake') 553 'bitbake')
554
500 if not is_merged_attr: 555 if not is_merged_attr:
501 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' 556 self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
502 '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' 557 f'{bitbake} {bbtargets} -u toasterui --read {confpath} --read {bblayers} --read {toasterlayers} --token="" >>{log} 2>&1;'
503 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ 558 f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'],
504 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
505 self.be.bbport, bitbake,)],
506 builddir, nowait=True) 559 builddir, nowait=True)
507 else: 560 else:
508 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' 561 self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
509 '%s %s -u toasterui --token="" >>%s 2>&1;' 562 f'{local_bitbake} {bbtargets} -u toasterui --token="" >>{log} 2>&1;'
510 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ 563 f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'],
511 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
512 self.be.bbport, bitbake,)],
513 builddir, nowait=True) 564 builddir, nowait=True)
514 565
515 logger.debug('localhostbecontroller: Build launched, exiting. ' 566 logger.debug('localhostbecontroller: Build launched, exiting. '
516 'Follow build logs at %s' % log) 567 f'Follow build logs at {log}')