#!/usr/bin/env python3 # # SPDX-License-Identifier: GPL-2.0-only # import logging import os import sys import argparse import warnings import json import shutil import time import stat import tempfile import configparser import datetime import glob import subprocess default_registry = 'git://github.com/kanavin/bitbake-setup-configurations.git;protocol=https;branch=main;rev=main' bindir = os.path.abspath(os.path.dirname(__file__)) sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')] import bb.msg import bb.process logger = bb.msg.logger_create('bitbake-setup', sys.stdout) def cache_dir(top_dir): return os.path.join(top_dir, '.bitbake-setup-cache') def init_bb_cache(settings, args): dldir = settings["default"]["dl-dir"] bb_cachedir = os.path.join(cache_dir(args.top_dir), 'bitbake-cache') d = bb.data.init() d.setVar("DL_DIR", dldir) d.setVar("BB_CACHEDIR", bb_cachedir) d.setVar("__BBSRCREV_SEEN", "1") if args.no_network: d.setVar("BB_SRCREV_POLICY", "cache") bb.fetch.fetcher_init(d) return d def save_bb_cache(): bb.fetch2.fetcher_parse_save() bb.fetch2.fetcher_parse_done() def get_config_name(config): return os.path.basename(config).split('.')[0] def copy_and_commit_config(config_path, dest_config_dir): shutil.copy(config_path, dest_config_dir) bb.process.run("git -C {} add .".format(dest_config_dir)) bb.process.run("git -C {} commit -a -m 'Configuration at {}'".format(dest_config_dir, time.asctime())) def _write_layer_list(dest, repodirs): layers = [] for r in repodirs: for root, dirs, files in os.walk(os.path.join(dest,r)): if os.path.basename(root) == 'conf' and 'layer.conf' in files: layers.append(os.path.relpath(os.path.dirname(root), dest)) layers_f = os.path.join(dest, ".oe-layers.json") with open(layers_f, 'w') as f: json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4) def checkout_layers(layers, layerdir, d): repodirs = [] oesetupbuild = None for r_name in layers: r_data = layers[r_name] repodir = r_data["path"] repodirs.append(repodir) r_remote = r_data['git-remote'] rev = r_remote['rev'] remotes = r_remote['remotes'] for remote in remotes: type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"]) fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) print("Fetching layer/tool repository {} into {}".format(r_name, os.path.join(layerdir,repodir))) fetcher = bb.fetch.Fetch(["{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir)], d) do_fetch(fetcher, layerdir) if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')): oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build') oeinitbuildenv = os.path.join(layerdir, repodir, 'oe-init-build-env') _write_layer_list(layerdir, repodirs) if oesetupbuild: links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'init-build-env': oeinitbuildenv} for l,t in links.items(): symlink = os.path.join(layerdir, l) if os.path.lexists(symlink): os.remove(symlink) os.symlink(os.path.relpath(t,layerdir),symlink) def setup_bitbake_build(name, bitbake_config, layerdir, builddir): def _setup_build_conf(layers, build_conf_dir): os.makedirs(build_conf_dir) layers_s = "\n".join([" {} \\".format(os.path.join(layerdir,l)) for l in layers]) bblayers_conf = """BBLAYERS ?= " \ {} " """.format(layers_s) with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f: f.write(bblayers_conf) local_conf = """# # This file is intended for local configuration tweaks. # # If you would like to publish and share changes made to this file, # it is recommended to put them into a distro config, or to create # layer fragments from changes made here. # """ with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f: f.write(local_conf) with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f: f.write("") with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f: f.write(bitbake_config["description"] + "\n") with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f: f.write("") def _make_init_build_env(builddir, initbuildenv): cmd = ". {} {}".format(initbuildenv, builddir) initbuild_in_builddir = os.path.join(builddir, 'init-build-env') with open(initbuild_in_builddir, 'w') as f: f.write(cmd) bitbake_builddir = os.path.join(builddir, "build-{}".format(name)) print("==============================") print("Setting up bitbake configuration {} in {}".format(name, bitbake_builddir)) template = bitbake_config.get("oe-template") layers = bitbake_config.get("bb-layers") if not template and not layers: print("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.") return oesetupbuild = os.path.join(layerdir, 'setup-build') if template and not os.path.exists(oesetupbuild): raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template)) bitbake_confdir = os.path.join(bitbake_builddir, 'conf') backup_bitbake_confdir = bitbake_confdir + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) if os.path.exists(bitbake_confdir): os.rename(bitbake_confdir, backup_bitbake_confdir) if layers: _setup_build_conf(layers, bitbake_confdir) if template: bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir)) else: initbuildenv = os.path.join(layerdir, 'init-build-env') if not os.path.exists(initbuildenv): print("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment") return _make_init_build_env(bitbake_builddir, os.path.realpath(initbuildenv)) build_script = os.path.join(bitbake_builddir, "build-targets") init_script = os.path.join(bitbake_builddir, "init-build-env") targets = " && ".join(bitbake_config["targets"]) shell = os.path.basename(os.environ.get("SHELL","bash")) with open(build_script,'w') as f: f.write("#!/usr/bin/env {}\n. {} && {}\n".format(shell, init_script, targets)) st = os.stat(build_script) os.chmod(build_script, st.st_mode | stat.S_IEXEC) fragments = bitbake_config.get("oe-fragments") if fragments: bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments))) if os.path.exists(backup_bitbake_confdir): bitbake_config_diff = get_diff(backup_bitbake_confdir, bitbake_confdir) if bitbake_config_diff: print("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir)) print("The bitbake configuration has changed:") print(bitbake_config_diff) else: shutil.rmtree(backup_bitbake_confdir) print("This bitbake configuration provides: {}".format(bitbake_config["description"])) readme = """{}\n\nAdditional information is in {} and {}\n Run {} to execute the default build targets for this bitbake configuration. Source the environment using '. {}' to run builds from the command line. The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf """.format( bitbake_config["description"], os.path.join(bitbake_builddir,'conf/conf-summary.txt'), os.path.join(bitbake_builddir,'conf/conf-notes.txt'), build_script, init_script, bitbake_builddir ) readme_file = os.path.join(bitbake_builddir, "README") with open(readme_file, 'w') as f: f.write(readme) print("Usage instructions and additional information are in {}".format(readme_file)) def get_registry_config(registry_path, id, dest_dir): for root, dirs, files in os.walk(registry_path): for f in files: if f.endswith('.conf.json') and id == get_config_name(f): shutil.copy(os.path.join(root, f), dest_dir) return f raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id)) def obtain_config(upstream_config, dest_dir, cache_dir, d): if upstream_config["type"] == 'local': shutil.copy(upstream_config['path'], dest_dir) basename = os.path.basename(upstream_config['path']) elif upstream_config["type"] == 'network': bb.process.run("wget {}".format(upstream_config["uri"]), cwd=dest_dir) basename = os.path.basename(upstream_config['uri']) elif upstream_config["type"] == 'registry': registry_path = update_registry(upstream_config["registry"], cache_dir, d) basename = get_registry_config(registry_path, upstream_config["id"], dest_dir) else: raise Exception("Unknown configuration type: {}".format(upstream_config["type"])) config_path = os.path.join(dest_dir, basename) config_data = json.load(open(config_path)) expiry_date = config_data.get("expires", None) if has_expired(expiry_date): print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date)) return config_path def update_build(config_path, confdir, builddir, layerdir, d, update_layers_only=False): json_data = json.load(open(config_path)) bitbake_configs = json_data["bitbake-setup"]["configuration"] layer_config = json_data["sources"] if not update_layers_only: copy_and_commit_config(config_path, confdir) checkout_layers(layer_config, layerdir, d) for bitbake_config_name, bitbake_config in bitbake_configs.items(): setup_bitbake_build(bitbake_config_name, bitbake_config, layerdir, builddir) def init_config(settings, args, d): stdout = sys.stdout def handle_task_progress(event, d): rate = event.rate if event.rate else '' progress = event.progress if event.progress > 0 else 0 print("{}% {} ".format(progress, rate), file=stdout, end='\r') config_name = get_config_name(args.config) builddir = os.path.join(os.path.abspath(args.top_dir), config_name) if os.path.exists(builddir): print("Build already initialized in {}\nUse 'bitbake-setup status' to check if it needs to be updated or 'bitbake-setup update' to perform the update.".format(builddir)) return print("Initializing a {} build in {}".format(config_name, builddir)) if os.path.exists(args.config): upstream_config = {'type':'local','path':os.path.abspath(args.config)} elif args.config.startswith("http://") or args.config.startswith("https://"): upstream_config = {'type':'network','uri':args.config} else: upstream_config = {'type':'registry','registry':settings["default"]["registry"],'id':args.config} os.makedirs(builddir) with open(os.path.join(builddir, "config-upstream.json"),'w') as s: json.dump(upstream_config, s, sort_keys=True, indent=4) confdir = os.path.join(builddir, "config") layerdir = os.path.join(builddir, "layers") os.makedirs(confdir) os.makedirs(layerdir) bb.process.run("git -C {} init -b main".format(confdir)) bb.process.run("git -C {} commit --allow-empty -m 'Initial commit'".format(confdir)) bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d) with tempfile.TemporaryDirectory(dir=builddir, prefix='config-tmp-') as tmpdirname: config_path = obtain_config(upstream_config, tmpdirname, cache_dir(args.top_dir), d) update_build(config_path, confdir, builddir, layerdir, d) bb.event.remove("bb.build.TaskProgress", None) def get_diff(file1, file2): try: bb.process.run('diff -uNr {} {}'.format(file1, file2)) except bb.process.ExecutionError as e: if e.exitcode == 1: return e.stdout else: raise e return None def are_layers_changed(layers, layerdir, d): changed = False for r_name in layers: r_data = layers[r_name] repodir = r_data["path"] r_remote = r_data['git-remote'] rev = r_remote['rev'] remotes = r_remote['remotes'] for remote in remotes: type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"]) fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d) upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default') rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir))) local_revision = rev_parse_result[0].strip() if upstream_revision != local_revision: changed = True print('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remotes[remote]["uri"], os.path.join(layerdir, repodir), rev, local_revision, upstream_revision)) return changed def build_status(settings, args, d, update=False): builddir = args.build_dir confdir = os.path.join(builddir, "config") layerdir = os.path.join(builddir, "layers") upstream_config = json.load(open(os.path.join(builddir, "config-upstream.json"))) with tempfile.TemporaryDirectory(dir=builddir, prefix='config-tmp-') as tmpdirname: current_config_path = obtain_config(upstream_config, tmpdirname, cache_dir(args.top_dir), d) build_config_path = os.path.join(confdir, os.path.basename(current_config_path)) config_diff = get_diff(build_config_path, current_config_path) if config_diff: print('Configuration in {} has changed:\n{}'.format(builddir, config_diff)) if update: update_build(current_config_path, confdir, builddir, layerdir, d) return if are_layers_changed(json.load(open(build_config_path))["sources"], layerdir, d): if update: update_build(build_config_path, confdir, builddir, layerdir, d, update_layers_only=True) return print("Configuration in {} has not changed.".format(builddir)) def build_update(settings, args, d): build_status(settings, args, d, update=True) def do_fetch(fetcher, dir): # git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch # and we need to set up smth similar here fetchlogdir = os.path.join(dir, 'logs') os.makedirs(fetchlogdir, exist_ok=True) fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))) with open(fetchlog, 'a') as f: oldstdout = sys.stdout sys.stdout = f fetcher.download() fetcher.unpack(dir) sys.stdout = oldstdout def update_registry(registry, cachedir, d): registrydir = 'configurations' full_registrydir = os.path.join(cachedir, registrydir) print("Fetching configuration registry {} into {}".format(registry, full_registrydir)) fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d) do_fetch(fetcher, cachedir) return full_registrydir def has_expired(expiry_date): if expiry_date: return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date) return False def list_registry(registry_path, with_expired): json_data = {} print("\nAvailable configurations:") for root, dirs, files in os.walk(registry_path): for f in files: if f.endswith('.conf.json'): config_name = get_config_name(f) config_data = json.load(open(os.path.join(root, f))) config_desc = config_data["description"] expiry_date = config_data.get("expires", None) if expiry_date: if with_expired or not has_expired(expiry_date): print("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date)) json_data[config_name] = {"description": config_desc, "expires": expiry_date} else: print("{}\t{}".format(config_name, config_desc)) json_data[config_name] = {"description": config_desc} print("\nRun 'init' with one of the above configuration identifiers to set up a build.") return json_data def list_configs(settings, args, d): registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d) json_data = list_registry(registry_path, args.with_expired) if args.write_json: with open(args.write_json, 'w') as f: json.dump(json_data, f, sort_keys=True, indent=4) print("Available configurations written into {}".format(args.write_json)) def install_buildtools(settings, args, d): buildtools_install_dir = os.path.join(args.build_dir, 'buildtools') if os.path.exists(buildtools_install_dir): if not args.force: print("Buildtools are already installed in {}.".format(buildtools_install_dir)) env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*')) if env_scripts: print("If you wish to use them, you need to source the the environment setup script e.g.") for s in env_scripts: print("$ . {}".format(s)) print("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.") return shutil.rmtree(buildtools_install_dir) install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools') buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S"))) print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir)) subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True) def default_settings_path(top_dir): return os.path.join(top_dir, 'bitbake-setup.conf') def write_settings(top_dir, force_replace): settings_path = default_settings_path(top_dir) if not os.path.exists(settings_path) or force_replace: if os.path.exists(settings_path): backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) os.rename(settings_path, backup_conf) print("Previous settings are in {}".format(backup_conf)) settings = configparser.ConfigParser() settings['default'] = { 'registry':default_registry, 'dl-dir':os.path.join(top_dir, '.bitbake-setup-downloads'), } os.makedirs(os.path.dirname(settings_path), exist_ok=True) with open(settings_path, 'w') as settingsfile: settings.write(settingsfile) print('Created a new settings file in {}.\n'.format(settings_path)) def load_settings(top_dir): # This creates a new settings file if it does not yet exist write_settings(top_dir, force_replace=False) settings_path = default_settings_path(top_dir) settings = configparser.ConfigParser() print('Loading settings from {}\n'.format(settings_path)) settings.read([settings_path]) return settings def change_settings(top_dir, new_settings): # This creates a new settings file if it does not yet exist write_settings(top_dir, force_replace=False) settings = load_settings(top_dir) for section, section_settings in new_settings.items(): for setting, value in section_settings.items(): settings[section][setting] = value print("Setting '{}' in section '{}' is changed to '{}'".format(setting, section, value)) settings_path = default_settings_path(top_dir) with open(settings_path, 'w') as settingsfile: settings.write(settingsfile) print("New settings written to {}".format(settings_path)) return settings def get_build_dir_via_bbpath(): bbpath = os.environ.get('BBPATH') if bbpath: build_dir = os.path.dirname(os.path.normpath(bbpath.split(':')[0])) if os.path.exists(os.path.join(build_dir,'config-upstream.json')): return build_dir return None def get_default_top_dir(): build_dir_via_bbpath = get_build_dir_via_bbpath() if build_dir_via_bbpath: top_dir = os.path.dirname(build_dir_via_bbpath) if os.path.exists(default_settings_path(top_dir)): return top_dir return os.path.join(os.path.expanduser('~'), 'bitbake-builds') def main(): def add_top_dir_arg(parser): parser.add_argument('--top-dir', default=get_default_top_dir(), help='Top level directory where builds are set up and downloaded configurations and layers are cached for reproducibility and offline builds, default is %(default)s') def add_build_dir_arg(parser): build_dir = get_build_dir_via_bbpath() if build_dir: parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH") else: parser.add_argument('--build-dir', required=True, help="Path to the build") parser = argparse.ArgumentParser( description="BitBake setup utility. Run with 'list' argument to get started.", epilog="Use %(prog)s --help to get help on a specific command" ) parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.') subparsers = parser.add_subparsers() parser_list = subparsers.add_parser('list', help='List available configurations') add_top_dir_arg(parser_list) parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.') parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.') parser_list.set_defaults(func=list_configs) parser_init = subparsers.add_parser('init', help='Initialize a build from a configuration') add_top_dir_arg(parser_init) parser_init.add_argument('config', help="path/URL/id to a configuration file, use 'list' command to get available ids") parser_init.set_defaults(func=init_config) parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration') add_build_dir_arg(parser_status) parser_status.set_defaults(func=build_status) parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration') add_build_dir_arg(parser_update) parser_update.set_defaults(func=build_update) parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine') add_build_dir_arg(parser_install_buildtools) parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.') parser_install_buildtools.set_defaults(func=install_buildtools) parser_reset_settings = subparsers.add_parser('reset-settings', help='Write a settings file with default values into the top level directory (contains the location of build configuration registry, downloads directory and other global settings)') add_top_dir_arg(parser_reset_settings) parser_reset_settings.set_defaults(func=write_settings) parser_change_setting = subparsers.add_parser('change-setting', help='Change a setting in the settings file') add_top_dir_arg(parser_change_setting) parser_change_setting.add_argument('section', help="Section in a settings file, typically 'default'") parser_change_setting.add_argument('key', help="Name of the setting") parser_change_setting.add_argument('value', help="Value of the setting") parser_change_setting.set_defaults(func=change_settings) args = parser.parse_args() logging.basicConfig(stream=sys.stdout) if args.debug: logger.setLevel(logging.DEBUG) elif args.quiet: logger.setLevel(logging.ERROR) # Need to re-run logger_create with color argument # (will be the same logger since it has the same name) bb.msg.logger_create('bitbake-setup', output=sys.stdout, color=args.color, level=logger.getEffectiveLevel()) if hasattr(args, 'build_dir'): if not os.path.exists(os.path.join(args.build_dir,'config-upstream.json')): print("Not a valid build directory: config-upstream.json does not exist in {}".format(args.build_dir)) return # commands without --top-dir argument (status, update) still need to know where # the top dir is, but it should be auto-deduced as parent of args.build_dir if not hasattr(args, 'top_dir'): args.top_dir = os.path.dirname(os.path.normpath(args.build_dir)) if 'func' in args: if args.func == write_settings: write_settings(args.top_dir, force_replace=True) elif args.func == change_settings: change_settings(args.top_dir, {args.section:{args.key:args.value}}) else: settings = load_settings(args.top_dir) d = init_bb_cache(settings, args) args.func(settings, args, d) save_bb_cache() else: from argparse import Namespace parser.print_help() main()