summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster/bldcontrol
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/toaster/bldcontrol')
-rw-r--r--bitbake/lib/toaster/bldcontrol/__init__.py0
-rw-r--r--bitbake/lib/toaster/bldcontrol/admin.py11
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py126
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py516
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/__init__.py0
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/__init__.py0
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py170
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py275
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py113
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py19
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py19
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py34
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py19
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py19
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py29
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py48
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/__init__.py0
-rw-r--r--bitbake/lib/toaster/bldcontrol/models.py165
-rw-r--r--bitbake/lib/toaster/bldcontrol/views.py5
19 files changed, 0 insertions, 1568 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/__init__.py b/bitbake/lib/toaster/bldcontrol/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/bitbake/lib/toaster/bldcontrol/__init__.py
+++ /dev/null
diff --git a/bitbake/lib/toaster/bldcontrol/admin.py b/bitbake/lib/toaster/bldcontrol/admin.py
deleted file mode 100644
index 1754bc11cc..0000000000
--- a/bitbake/lib/toaster/bldcontrol/admin.py
+++ /dev/null
@@ -1,11 +0,0 @@
1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5from django.contrib import admin
6from .models import BuildEnvironment
7
8class BuildEnvironmentAdmin(admin.ModelAdmin):
9 pass
10
11admin.site.register(BuildEnvironment, BuildEnvironmentAdmin)
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
deleted file mode 100644
index 71c288df34..0000000000
--- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ /dev/null
@@ -1,126 +0,0 @@
1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2014 Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9import os
10import sys
11from django.db.models import Q
12from bldcontrol.models import BuildEnvironment, BRLayer, BRBitbake
13
14# load Bitbake components
15path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
16sys.path.insert(0, path)
17
18class BitbakeController(object):
19 """ This is the basic class that controlls a bitbake server.
20 It is outside the scope of this class on how the server is started and aquired
21 """
22
23 def __init__(self, be):
24 import bb.server.xmlrpcclient
25 self.connection = bb.server.xmlrpcclient._create_server(be.bbaddress,
26 int(be.bbport))[0]
27
28 def _runCommand(self, command):
29 result, error = self.connection.runCommand(command)
30 if error:
31 raise Exception(error)
32 return result
33
34 def disconnect(self):
35 return self.connection.removeClient()
36
37 def setVariable(self, name, value):
38 return self._runCommand(["setVariable", name, value])
39
40 def getVariable(self, name):
41 return self._runCommand(["getVariable", name])
42
43 def triggerEvent(self, event):
44 return self._runCommand(["triggerEvent", event])
45
46 def build(self, targets, task = None):
47 if task is None:
48 task = "build"
49 return self._runCommand(["buildTargets", targets, task])
50
51 def forceShutDown(self):
52 return self._runCommand(["stateForceShutdown"])
53
54
55
56def getBuildEnvironmentController(**kwargs):
57 """ Gets you a BuildEnvironmentController that encapsulates a build environment,
58 based on the query dictionary sent in.
59
60 This is used to retrieve, for example, the currently running BE from inside
61 the toaster UI, or find a new BE to start a new build in it.
62
63 The return object MUST always be a BuildEnvironmentController.
64 """
65
66 from bldcontrol.localhostbecontroller import LocalhostBEController
67
68 be = BuildEnvironment.objects.filter(Q(**kwargs))[0]
69 if be.betype == BuildEnvironment.TYPE_LOCAL:
70 return LocalhostBEController(be)
71 else:
72 raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
73
74
75class BuildEnvironmentController(object):
76 """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST
77 or SHOULD be supported by a Build Environment. It is used to establish the framework, and must
78 not be instantiated directly by the user.
79
80 Use the "getBuildEnvironmentController()" function to get a working BEC for your remote.
81
82 How the BuildEnvironments are discovered is outside the scope of this class.
83
84 You must derive this class to teach Toaster how to operate in your own infrastructure.
85 We provide some specific BuildEnvironmentController classes that can be used either to
86 directly set-up Toaster infrastructure, or as a model for your own infrastructure set:
87
88 * Localhost controller will run the Toaster BE on the same account as the web server
89 (current user if you are using the the Django development web server)
90 on the local machine, with the "build/" directory under the "poky/" source checkout directory.
91 Bash is expected to be available.
92
93 """
94 def __init__(self, be):
95 """ Takes a BuildEnvironment object as parameter that points to the settings of the BE.
96 """
97 self.be = be
98 self.connection = None
99
100 def setLayers(self, bitbake, ls):
101 """ Checks-out bitbake executor and layers from git repositories.
102 Sets the layer variables in the config file, after validating local layer paths.
103 bitbake must be a single BRBitbake instance
104 The layer paths must be in a list of BRLayer object
105
106 a word of attention: by convention, the first layer for any build will be poky!
107 """
108 raise NotImplementedError("FIXME: Must override setLayers")
109
110 def getArtifact(self, path):
111 """ This call returns an artifact identified by the 'path'. How 'path' is interpreted as
112 up to the implementing BEC. The return MUST be a REST URL where a GET will actually return
113 the content of the artifact, e.g. for use as a "download link" in a web UI.
114 """
115 raise NotImplementedError("Must return the REST URL of the artifact")
116
117 def triggerBuild(self, bitbake, layers, variables, targets):
118 raise NotImplementedError("Must override BE release")
119
120class ShellCmdException(Exception):
121 pass
122
123
124class BuildSetupException(Exception):
125 pass
126
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
deleted file mode 100644
index 577e765f11..0000000000
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ /dev/null
@@ -1,516 +0,0 @@
1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2014 Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9import os
10import re
11import shutil
12import time
13from bldcontrol.models import BuildEnvironment, BuildRequest, Build
14from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ToasterSetting
15from orm.models import signal_runbuilds
16import subprocess
17
18from toastermain import settings
19
20from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException
21
22import logging
23logger = logging.getLogger("toaster")
24
25install_dir = os.environ.get('TOASTER_DIR')
26
27from pprint import pformat
28
29class LocalhostBEController(BuildEnvironmentController):
30 """ Implementation of the BuildEnvironmentController for the localhost;
31 this controller manages the default build directory,
32 the server setup and system start and stop for the localhost-type build environment
33
34 """
35
36 def __init__(self, be):
37 super(LocalhostBEController, self).__init__(be)
38 self.pokydirname = None
39 self.islayerset = False
40
41 def _shellcmd(self, command, cwd=None, nowait=False,env=None):
42 if cwd is None:
43 cwd = self.be.sourcedir
44 if env is None:
45 env=os.environ.copy()
46
47 logger.debug("lbc_shellcmd: (%s) %s" % (cwd, command))
48 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
49 if nowait:
50 return
51 (out,err) = p.communicate()
52 p.wait()
53 if p.returncode:
54 if len(err) == 0:
55 err = "command: %s \n%s" % (command, out)
56 else:
57 err = "command: %s \n%s" % (command, err)
58 logger.warning("localhostbecontroller: shellcmd error %s" % err)
59 raise ShellCmdException(err)
60 else:
61 logger.debug("localhostbecontroller: shellcmd success")
62 return out.decode('utf-8')
63
64 def getGitCloneDirectory(self, url, branch):
65 """Construct unique clone directory name out of url and branch."""
66 if branch != "HEAD":
67 return "_toaster_clones/_%s_%s" % (re.sub('[:/@+%]', '_', url), branch)
68
69 # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
70 # which _ALWAYS_ means the current poky checkout
71 from os.path import dirname as DN
72 local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__))))))
73 #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
74 return local_checkout_path
75
76 def setCloneStatus(self,bitbake,status,total,current,repo_name):
77 bitbake.req.build.repos_cloned=current
78 bitbake.req.build.repos_to_clone=total
79 bitbake.req.build.progress_item=repo_name
80 bitbake.req.build.save()
81
82 def setLayers(self, bitbake, layers, targets):
83 """ a word of attention: by convention, the first layer for any build will be poky! """
84
85 assert self.be.sourcedir is not None
86
87 layerlist = []
88 nongitlayerlist = []
89 layer_index = 0
90 git_env = os.environ.copy()
91 # (note: add custom environment settings here)
92
93 # set layers in the layersource
94
95 # 1. get a list of repos with branches, and map dirpaths for each layer
96 gitrepos = {}
97
98 # if we're using a remotely fetched version of bitbake add its git
99 # details to the list of repos to clone
100 if bitbake.giturl and bitbake.commit:
101 gitrepos[(bitbake.giturl, bitbake.commit)] = []
102 gitrepos[(bitbake.giturl, bitbake.commit)].append(
103 ("bitbake", bitbake.dirpath, 0))
104
105 for layer in layers:
106 # We don't need to git clone the layer for the CustomImageRecipe
107 # as it's generated by us layer on if needed
108 if CustomImageRecipe.LAYER_NAME in layer.name:
109 continue
110
111 # If we have local layers then we don't need clone them
112 # For local layers giturl will be empty
113 if not layer.giturl:
114 nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) )
115 continue
116
117 if not (layer.giturl, layer.commit) in gitrepos:
118 gitrepos[(layer.giturl, layer.commit)] = []
119 gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) )
120 layer_index += 1
121
122
123 logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
124
125
126 # 2. Note for future use if the current source directory is a
127 # checked-out git repos that could match a layer's vcs_url and therefore
128 # be used to speed up cloning (rather than fetching it again).
129
130 cached_layers = {}
131
132 try:
133 for remotes in self._shellcmd("git remote -v", self.be.sourcedir,env=git_env).split("\n"):
134 try:
135 remote = remotes.split("\t")[1].split(" ")[0]
136 if remote not in cached_layers:
137 cached_layers[remote] = self.be.sourcedir
138 except IndexError:
139 pass
140 except ShellCmdException:
141 # ignore any errors in collecting git remotes this is an optional
142 # step
143 pass
144
145 logger.info("Using pre-checked out source for layer %s", cached_layers)
146
147 # 3. checkout the repositories
148 clone_count=0
149 clone_total=len(gitrepos.keys())
150 self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'')
151 for giturl, commit in gitrepos.keys():
152 self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0])
153 clone_count += 1
154
155 localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
156 logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
157
158 # see if our directory is a git repository
159 if os.path.exists(localdirname):
160 try:
161 localremotes = self._shellcmd("git remote -v",
162 localdirname,env=git_env)
163 # NOTE: this nice-to-have check breaks when using git remaping to get past firewall
164 # Re-enable later with .gitconfig remapping checks
165 #if not giturl in localremotes and commit != 'HEAD':
166 # raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
167 pass
168 except ShellCmdException:
169 # our localdirname might not be a git repository
170 #- that's fine
171 pass
172 else:
173 if giturl in cached_layers:
174 logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
175 self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname),env=git_env)
176 self._shellcmd("git remote remove origin", localdirname,env=git_env)
177 self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname,env=git_env)
178 else:
179 logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
180 self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname),env=git_env)
181
182 # branch magic name "HEAD" will inhibit checkout
183 if commit != "HEAD":
184 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
185 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
186 self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env)
187
188 # take the localdirname as poky dir if we can find the oe-init-build-env
189 if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
190 logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
191 self.pokydirname = localdirname
192
193 # make sure we have a working bitbake
194 if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
195 logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
196 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)
197
198 # verify our repositories
199 for name, dirpath, index in gitrepos[(giturl, commit)]:
200 localdirpath = os.path.join(localdirname, dirpath)
201 logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath)
202 if not os.path.exists(localdirpath):
203 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Exiting." % (localdirpath, giturl, commit))
204
205 if name != "bitbake":
206 layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/")))
207
208 self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'')
209 logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
210
211 # Resolve self.pokydirname if not resolved yet, consider the scenario
212 # where all layers are local, that's the else clause
213 if self.pokydirname is None:
214 if os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
215 logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
216 self.pokydirname = self.be.sourcedir
217 else:
218 # Alternatively, scan local layers for relative "oe-init-build-env" location
219 for layer in layers:
220 if os.path.exists(os.path.join(layer.layer_version.layer.local_source_dir,"..","oe-init-build-env")):
221 logger.debug("localhostbecontroller, setting pokydirname to %s" % (layer.layer_version.layer.local_source_dir))
222 self.pokydirname = os.path.join(layer.layer_version.layer.local_source_dir,"..")
223 break
224 else:
225 logger.error("pokydirname is not set, you will run into trouble!")
226
227 # 5. create custom layer and add custom recipes to it
228 for target in targets:
229 try:
230 customrecipe = CustomImageRecipe.objects.get(
231 name=target.target,
232 project=bitbake.req.project)
233
234 custom_layer_path = self.setup_custom_image_recipe(
235 customrecipe, layers)
236
237 if os.path.isdir(custom_layer_path):
238 layerlist.append("%03d:%s" % (layer_index,custom_layer_path))
239
240 except CustomImageRecipe.DoesNotExist:
241 continue # not a custom recipe, skip
242
243 layerlist.extend(nongitlayerlist)
244 logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
245 self.islayerset = True
246
247 # restore the order of layer list for bblayers.conf
248 layerlist.sort()
249 sorted_layerlist = [l[4:] for l in layerlist]
250 return sorted_layerlist
251
252 def setup_custom_image_recipe(self, customrecipe, layers):
253 """ Set up toaster-custom-images layer and recipe files """
254 layerpath = os.path.join(self.be.builddir,
255 CustomImageRecipe.LAYER_NAME)
256
257 # create directory structure
258 for name in ("conf", "recipes"):
259 path = os.path.join(layerpath, name)
260 if not os.path.isdir(path):
261 os.makedirs(path)
262
263 # create layer.conf
264 config = os.path.join(layerpath, "conf", "layer.conf")
265 if not os.path.isfile(config):
266 with open(config, "w") as conf:
267 conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
268
269 # Update the Layer_Version dirpath that has our base_recipe in
270 # to be able to read the base recipe to then generate the
271 # custom recipe.
272 br_layer_base_recipe = layers.get(
273 layer_version=customrecipe.base_recipe.layer_version)
274
275 # If the layer is one that we've cloned we know where it lives
276 if br_layer_base_recipe.giturl and br_layer_base_recipe.commit:
277 layer_path = self.getGitCloneDirectory(
278 br_layer_base_recipe.giturl,
279 br_layer_base_recipe.commit)
280 # Otherwise it's a local layer
281 elif br_layer_base_recipe.local_source_dir:
282 layer_path = br_layer_base_recipe.local_source_dir
283 else:
284 logger.error("Unable to workout the dir path for the custom"
285 " image recipe")
286
287 br_layer_base_dirpath = os.path.join(
288 self.be.sourcedir,
289 layer_path,
290 customrecipe.base_recipe.layer_version.dirpath)
291
292 customrecipe.base_recipe.layer_version.dirpath = br_layer_base_dirpath
293
294 customrecipe.base_recipe.layer_version.save()
295
296 # create recipe
297 recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
298 customrecipe.name)
299 with open(recipe_path, "w") as recipef:
300 recipef.write(customrecipe.generate_recipe_file_contents())
301
302 # Update the layer and recipe objects
303 customrecipe.layer_version.dirpath = layerpath
304 customrecipe.layer_version.layer.local_source_dir = layerpath
305 customrecipe.layer_version.layer.save()
306 customrecipe.layer_version.save()
307
308 customrecipe.file_path = recipe_path
309 customrecipe.save()
310
311 return layerpath
312
313
314 def readServerLogFile(self):
315 return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
316
317
318 def triggerBuild(self, bitbake, layers, variables, targets, brbe):
319 layers = self.setLayers(bitbake, layers, targets)
320 is_merged_attr = bitbake.req.project.merged_attr
321
322 git_env = os.environ.copy()
323 # (note: add custom environment settings here)
324 try:
325 # insure that the project init/build uses the selected bitbake, and not Toaster's
326 del git_env['TEMPLATECONF']
327 del git_env['BBBASEDIR']
328 del git_env['BUILDDIR']
329 except KeyError:
330 pass
331
332 # init build environment from the clone
333 if bitbake.req.project.builddir:
334 builddir = bitbake.req.project.builddir
335 else:
336 builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
337 oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
338 # init build environment
339 try:
340 custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
341 custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
342 self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
343 except ToasterSetting.DoesNotExist:
344 self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
345 self.be.sourcedir,env=git_env)
346
347 # update bblayers.conf
348 if not is_merged_attr:
349 bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
350 with open(bblconfpath, 'w') as bblayers:
351 bblayers.write('# line added by toaster build control\n'
352 'BBLAYERS = "%s"' % ' '.join(layers))
353
354 # write configuration file
355 confpath = os.path.join(builddir, 'conf/toaster.conf')
356 with open(confpath, 'w') as conf:
357 for var in variables:
358 conf.write('%s="%s"\n' % (var.name, var.value))
359 conf.write('INHERIT+="toaster buildhistory"')
360 else:
361 # Append the Toaster-specific values directly to the bblayers.conf
362 bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
363 bblconfpath_save = os.path.join(builddir, "conf/bblayers.conf.save")
364 shutil.copyfile(bblconfpath, bblconfpath_save)
365 with open(bblconfpath) as bblayers:
366 content = bblayers.readlines()
367 do_write = True
368 was_toaster = False
369 with open(bblconfpath,'w') as bblayers:
370 for line in content:
371 #line = line.strip('\n')
372 if 'TOASTER_CONFIG_PROLOG' in line:
373 do_write = False
374 was_toaster = True
375 elif 'TOASTER_CONFIG_EPILOG' in line:
376 do_write = True
377 elif do_write:
378 bblayers.write(line)
379 if not was_toaster:
380 bblayers.write('\n')
381 bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n')
382 bblayers.write('BBLAYERS = "\\\n')
383 for layer in layers:
384 bblayers.write(' %s \\\n' % layer)
385 bblayers.write(' "\n')
386 bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n')
387 # Append the Toaster-specific values directly to the local.conf
388 bbconfpath = os.path.join(builddir, "conf/local.conf")
389 bbconfpath_save = os.path.join(builddir, "conf/local.conf.save")
390 shutil.copyfile(bbconfpath, bbconfpath_save)
391 with open(bbconfpath) as f:
392 content = f.readlines()
393 do_write = True
394 was_toaster = False
395 with open(bbconfpath,'w') as conf:
396 for line in content:
397 #line = line.strip('\n')
398 if 'TOASTER_CONFIG_PROLOG' in line:
399 do_write = False
400 was_toaster = True
401 elif 'TOASTER_CONFIG_EPILOG' in line:
402 do_write = True
403 elif do_write:
404 conf.write(line)
405 if not was_toaster:
406 conf.write('\n')
407 conf.write('#=== TOASTER_CONFIG_PROLOG ===\n')
408 for var in variables:
409 if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"):
410 conf.write('%s="%s"\n' % (var.name, var.value))
411 conf.write('#=== TOASTER_CONFIG_EPILOG ===\n')
412
413 # If 'target' is just the project preparation target, then we are done
414 for target in targets:
415 if "_PROJECT_PREPARE_" == target.target:
416 logger.debug('localhostbecontroller: Project has been prepared. Done.')
417 # Update the Build Request and release the build environment
418 bitbake.req.state = BuildRequest.REQ_COMPLETED
419 bitbake.req.save()
420 self.be.lock = BuildEnvironment.LOCK_FREE
421 self.be.save()
422 # Close the project build and progress bar
423 bitbake.req.build.outcome = Build.SUCCEEDED
424 bitbake.req.build.save()
425 # Update the project status
426 bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS)
427 signal_runbuilds()
428 return
429
430 # clean the Toaster to build environment
431 env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
432
433 # run bitbake server from the clone if available
434 # otherwise pick it from the PATH
435 bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
436 if not os.path.exists(bitbake):
437 logger.info("Bitbake not available under %s, will try to use it from PATH" %
438 self.pokydirname)
439 for path in os.environ["PATH"].split(os.pathsep):
440 if os.path.exists(os.path.join(path, 'bitbake')):
441 bitbake = os.path.join(path, 'bitbake')
442 break
443 else:
444 logger.error("Looks like Bitbake is not available, please fix your environment")
445
446 toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
447 if not is_merged_attr:
448 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
449 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
450 builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
451 else:
452 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s '
453 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
454 builddir, bitbake), self.be.sourcedir)
455
456 # read port number from bitbake.lock
457 self.be.bbport = -1
458 bblock = os.path.join(builddir, 'bitbake.lock')
459 # allow 10 seconds for bb lock file to appear but also be populated
460 for lock_check in range(10):
461 if not os.path.exists(bblock):
462 logger.debug("localhostbecontroller: waiting for bblock file to appear")
463 time.sleep(1)
464 continue
465 if 10 < os.stat(bblock).st_size:
466 break
467 logger.debug("localhostbecontroller: waiting for bblock content to appear")
468 time.sleep(1)
469 else:
470 raise BuildSetupException("Cannot find bitbake server lock file '%s'. Exiting." % bblock)
471
472 with open(bblock) as fplock:
473 for line in fplock:
474 if ":" in line:
475 self.be.bbport = line.split(":")[-1].strip()
476 logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
477 break
478
479 if -1 == self.be.bbport:
480 raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)
481
482 self.be.bbaddress = "localhost"
483 self.be.bbstate = BuildEnvironment.SERVER_STARTED
484 self.be.lock = BuildEnvironment.LOCK_RUNNING
485 self.be.save()
486
487 bbtargets = ''
488 for target in targets:
489 task = target.task
490 if task:
491 if not task.startswith('do_'):
492 task = 'do_' + task
493 task = ':%s' % task
494 bbtargets += '%s%s ' % (target.target, task)
495
496 # run build with local bitbake. stop the server after the build.
497 log = os.path.join(builddir, 'toaster_ui.log')
498 local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
499 'bitbake')
500 if not is_merged_attr:
501 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
502 '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;'
503 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
504 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
505 self.be.bbport, bitbake,)],
506 builddir, nowait=True)
507 else:
508 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
509 '%s %s -u toasterui --token="" >>%s 2>&1;'
510 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
511 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
512 self.be.bbport, bitbake,)],
513 builddir, nowait=True)
514
515 logger.debug('localhostbecontroller: Build launched, exiting. '
516 'Follow build logs at %s' % log)
diff --git a/bitbake/lib/toaster/bldcontrol/management/__init__.py b/bitbake/lib/toaster/bldcontrol/management/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/bitbake/lib/toaster/bldcontrol/management/__init__.py
+++ /dev/null
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py
+++ /dev/null
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
deleted file mode 100644
index 20f9dce569..0000000000
--- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
+++ /dev/null
@@ -1,170 +0,0 @@
1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5from django.core.management.base import BaseCommand
6
7from django.core.management import call_command
8from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
9from orm.models import ToasterSetting, Build, Layer
10
11import os
12import traceback
13import warnings
14
15
16def DN(path):
17 if path is None:
18 return ""
19 else:
20 return os.path.dirname(path)
21
22
23class Command(BaseCommand):
24 args = ""
25 help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings."
26
27 def __init__(self, *args, **kwargs):
28 super(Command, self).__init__(*args, **kwargs)
29 self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
30
31 def _verify_build_environment(self):
32 # provide a local build env. This will be extended later to include non local
33 if BuildEnvironment.objects.count() == 0:
34 BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
35
36 # we make sure we have builddir and sourcedir for all defined build envionments
37 for be in BuildEnvironment.objects.all():
38 be.needs_import = False
39 def _verify_be():
40 is_changed = False
41
42 def _update_sourcedir():
43 be.sourcedir = os.environ.get('TOASTER_DIR')
44 return True
45
46 if len(be.sourcedir) == 0:
47 is_changed = _update_sourcedir()
48
49 if not be.sourcedir.startswith("/"):
50 print("\n -- Validation: The layers checkout directory must be set to an absolute path.")
51 is_changed = _update_sourcedir()
52
53 if is_changed:
54 if be.betype == BuildEnvironment.TYPE_LOCAL:
55 be.needs_import = True
56 return True
57
58 def _update_builddir():
59 be.builddir = os.environ.get('TOASTER_DIR')+"/build"
60 return True
61
62 if len(be.builddir) == 0:
63 is_changed = _update_builddir()
64
65 if not be.builddir.startswith("/"):
66 print("\n -- Validation: The build directory must to be set to an absolute path.")
67 is_changed = _update_builddir()
68
69 if is_changed:
70 print("\nBuild configuration saved")
71 be.save()
72 return True
73
74 if be.needs_import:
75 try:
76 print("Loading default settings")
77 call_command("loaddata", "settings")
78 template_conf = os.environ.get("TEMPLATECONF", "")
79 custom_xml_only = os.environ.get("CUSTOM_XML_ONLY")
80
81 if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or custom_xml_only is not None:
82 # only use the custom settings
83 pass
84 elif "poky" in template_conf:
85 print("Loading poky configuration")
86 call_command("loaddata", "poky")
87 else:
88 print("Loading OE-Core configuration")
89 call_command("loaddata", "oe-core")
90 if template_conf:
91 oe_core_path = os.path.realpath(
92 template_conf +
93 "/../")
94 else:
95 print("TEMPLATECONF not found. You may have to"
96 " manually configure layer paths")
97 oe_core_path = input("Please enter the path of"
98 " your openembedded-core "
99 "layer: ")
100 # Update the layer instances of openemebedded-core
101 for layer in Layer.objects.filter(
102 name="openembedded-core",
103 local_source_dir="OE-CORE-LAYER-DIR"):
104 layer.local_path = oe_core_path
105 layer.save()
106
107 # Import the custom fixture if it's present
108 with warnings.catch_warnings():
109 warnings.filterwarnings(
110 action="ignore",
111 message="^.*No fixture named.*$")
112 print("Importing custom settings if present")
113 try:
114 call_command("loaddata", "custom")
115 except:
116 print("NOTE: optional fixture 'custom' not found")
117
118 # we run lsupdates after config update
119 print("\nFetching information from the layer index, "
120 "please wait.\nYou can re-update any time later "
121 "by running bitbake/lib/toaster/manage.py "
122 "lsupdates\n")
123 call_command("lsupdates")
124
125 # we don't look for any other config files
126 return is_changed
127 except Exception as e:
128 print("Failure while trying to setup toaster: %s"
129 % e)
130 traceback.print_exc()
131
132 return is_changed
133
134 while _verify_be():
135 pass
136 return 0
137
138 def _verify_default_settings(self):
139 # verify that default settings are there
140 if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1:
141 ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete()
142 ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='')
143 return 0
144
145 def _verify_builds_in_progress(self):
146 # we are just starting up. we must not have any builds in progress, or build environments taken
147 for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
148 BRError.objects.create(req=b, errtype="toaster",
149 errmsg=
150 "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed")
151
152 BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED)
153
154 BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE)
155
156 # also mark "In Progress builds as failures"
157 from django.utils import timezone
158 Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now())
159
160 return 0
161
162
163
164 def handle(self, **options):
165 retval = 0
166 retval += self._verify_build_environment()
167 retval += self._verify_default_settings()
168 retval += self._verify_builds_in_progress()
169
170 return retval
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
deleted file mode 100644
index 834e32b36f..0000000000
--- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ /dev/null
@@ -1,275 +0,0 @@
1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5from django.core.management.base import BaseCommand
6from django.db import transaction
7from django.db.models import Q
8
9from bldcontrol.bbcontroller import getBuildEnvironmentController
10from bldcontrol.models import BuildRequest, BuildEnvironment
11from bldcontrol.models import BRError, BRVariable
12
13from orm.models import Build, LogMessage, Target
14
15import logging
16import traceback
17import signal
18import os
19
20logger = logging.getLogger("toaster")
21
22
23class Command(BaseCommand):
24 args = ""
25 help = "Schedules and executes build requests as possible. "\
26 "Does not return (interrupt with Ctrl-C)"
27
28 @transaction.atomic
29 def _selectBuildEnvironment(self):
30 bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
31 bec.be.lock = BuildEnvironment.LOCK_LOCK
32 bec.be.save()
33 return bec
34
35 @transaction.atomic
36 def _selectBuildRequest(self):
37 br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
38 return br
39
40 def schedule(self):
41 try:
42 # select the build environment and the request to build
43 br = self._selectBuildRequest()
44 if br:
45 br.state = BuildRequest.REQ_INPROGRESS
46 br.save()
47 else:
48 return
49
50 try:
51 bec = self._selectBuildEnvironment()
52 except IndexError as e:
53 # we could not find a BEC; postpone the BR
54 br.state = BuildRequest.REQ_QUEUED
55 br.save()
56 logger.debug("runbuilds: No build env (%s)" % e)
57 return
58
59 logger.info("runbuilds: starting build %s, environment %s" %
60 (br, bec.be))
61
62 # let the build request know where it is being executed
63 br.environment = bec.be
64 br.save()
65
66 # this triggers an async build
67 bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
68 br.brvariable_set.all(), br.brtarget_set.all(),
69 "%d:%d" % (br.pk, bec.be.pk))
70
71 except Exception as e:
72 logger.error("runbuilds: Error launching build %s" % e)
73 traceback.print_exc()
74 if "[Errno 111] Connection refused" in str(e):
75 # Connection refused, read toaster_server.out
76 errmsg = bec.readServerLogFile()
77 else:
78 errmsg = str(e)
79
80 BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
81 traceback=traceback.format_exc())
82 br.state = BuildRequest.REQ_FAILED
83 br.save()
84 bec.be.lock = BuildEnvironment.LOCK_FREE
85 bec.be.save()
86 # Cancel the pending build and report the exception to the UI
87 log_object = LogMessage.objects.create(
88 build = br.build,
89 level = LogMessage.EXCEPTION,
90 message = errmsg)
91 log_object.save()
92 br.build.outcome = Build.FAILED
93 br.build.save()
94
95 def archive(self):
96 for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
97 if br.build is None:
98 br.state = BuildRequest.REQ_FAILED
99 else:
100 br.state = BuildRequest.REQ_COMPLETED
101 br.save()
102
103 def cleanup(self):
104 from django.utils import timezone
105 from datetime import timedelta
106 # environments locked for more than 30 seconds
107 # they should be unlocked
108 BuildEnvironment.objects.filter(
109 Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
110 BuildRequest.REQ_COMPLETED,
111 BuildRequest.REQ_CANCELLING]) &
112 Q(lock=BuildEnvironment.LOCK_LOCK) &
113 Q(updated__lt=timezone.now() - timedelta(seconds=30))
114 ).update(lock=BuildEnvironment.LOCK_FREE)
115
116 # update all Builds that were in progress and failed to start
117 for br in BuildRequest.objects.filter(
118 state=BuildRequest.REQ_FAILED,
119 build__outcome=Build.IN_PROGRESS):
120 # transpose the launch errors in ToasterExceptions
121 br.build.outcome = Build.FAILED
122 for brerror in br.brerror_set.all():
123 logger.debug("Saving error %s" % brerror)
124 LogMessage.objects.create(build=br.build,
125 level=LogMessage.EXCEPTION,
126 message=brerror.errmsg)
127 br.build.save()
128
129 # we don't have a true build object here; hence, toasterui
130 # didn't have a change to release the BE lock
131 br.environment.lock = BuildEnvironment.LOCK_FREE
132 br.environment.save()
133
134 # update all BuildRequests without a build created
135 for br in BuildRequest.objects.filter(build=None):
136 br.build = Build.objects.create(project=br.project,
137 completed_on=br.updated,
138 started_on=br.created)
139 br.build.outcome = Build.FAILED
140 try:
141 br.build.machine = br.brvariable_set.get(name='MACHINE').value
142 except BRVariable.DoesNotExist:
143 pass
144 br.save()
145 # transpose target information
146 for brtarget in br.brtarget_set.all():
147 Target.objects.create(build=br.build,
148 target=brtarget.target,
149 task=brtarget.task)
150 # transpose the launch errors in ToasterExceptions
151 for brerror in br.brerror_set.all():
152 LogMessage.objects.create(build=br.build,
153 level=LogMessage.EXCEPTION,
154 message=brerror.errmsg)
155
156 br.build.save()
157
158 # Make sure the LOCK is removed for builds which have been fully
159 # cancelled
160 for br in BuildRequest.objects.filter(
161 Q(build__outcome=Build.CANCELLED) &
162 Q(state=BuildRequest.REQ_CANCELLING) &
163 ~Q(environment=None)):
164 br.environment.lock = BuildEnvironment.LOCK_FREE
165 br.environment.save()
166
167 def runbuild(self):
168 try:
169 self.cleanup()
170 except Exception as e:
171 logger.warning("runbuilds: cleanup exception %s" % str(e))
172
173 try:
174 self.archive()
175 except Exception as e:
176 logger.warning("runbuilds: archive exception %s" % str(e))
177
178 try:
179 self.schedule()
180 except Exception as e:
181 logger.warning("runbuilds: schedule exception %s" % str(e))
182
183 # Test to see if a build pre-maturely died due to a bitbake crash
184 def check_dead_builds(self):
185 do_cleanup = False
186 try:
187 for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
188 # Get the build directory
189 if br.project.builddir:
190 builddir = br.project.builddir
191 else:
192 builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
193 # Check log to see if there is a recent traceback
194 toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
195 test_file = os.path.join(builddir, '._toaster_check.txt')
196 os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
197 traceback_text = ''
198 is_traceback = False
199 with open(test_file,'r') as test_file_fd:
200 test_file_tail = test_file_fd.readlines()
201 for line in test_file_tail:
202 if line.startswith('Traceback (most recent call last):'):
203 traceback_text = line
204 is_traceback = True
205 elif line.startswith('NOTE: ToasterUI waiting for events'):
206 # Ignore any traceback before new build start
207 traceback_text = ''
208 is_traceback = False
209 elif line.startswith('Note: Toaster traceback auto-stop'):
210 # Ignore any traceback before this previous traceback catch
211 traceback_text = ''
212 is_traceback = False
213 elif is_traceback:
214 traceback_text += line
215 # Test the results
216 is_stop = False
217 if is_traceback:
218 # Found a traceback
219 errtype = 'Bitbake crash'
220 errmsg = 'Bitbake crash\n' + traceback_text
221 state = BuildRequest.REQ_FAILED
222 # Clean up bitbake files
223 bitbake_lock = os.path.join(builddir, 'bitbake.lock')
224 if os.path.isfile(bitbake_lock):
225 os.remove(bitbake_lock)
226 bitbake_sock = os.path.join(builddir, 'bitbake.sock')
227 if os.path.isfile(bitbake_sock):
228 os.remove(bitbake_sock)
229 if os.path.isfile(test_file):
230 os.remove(test_file)
231 # Add note to ignore this traceback on next check
232 os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
233 is_stop = True
234 # Add more tests here
235 #elif ...
236 # Stop the build request?
237 if is_stop:
238 brerror = BRError(
239 req = br,
240 errtype = errtype,
241 errmsg = errmsg,
242 traceback = traceback_text,
243 )
244 brerror.save()
245 br.state = state
246 br.save()
247 do_cleanup = True
248 # Do cleanup
249 if do_cleanup:
250 self.cleanup()
251 except Exception as e:
252 logger.error("runbuilds: Error in check_dead_builds %s" % e)
253
254 def handle(self, **options):
255 pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
256 ".runbuilds.pid")
257
258 with open(pidfile_path, 'w') as pidfile:
259 pidfile.write("%s" % os.getpid())
260
261 # Clean up any stale/failed builds from previous Toaster run
262 self.runbuild()
263
264 signal.signal(signal.SIGUSR1, lambda sig, frame: None)
265
266 while True:
267 sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
268 if sigset:
269 for sig in sigset:
270 # Consume each captured pending event
271 self.runbuild()
272 else:
273 # Check for build exceptions
274 self.check_dead_builds()
275
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py b/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py
deleted file mode 100644
index 7ee19f4745..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py
+++ /dev/null
@@ -1,113 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('orm', '0001_initial'),
11 ]
12
13 operations = [
14 migrations.CreateModel(
15 name='BRBitbake',
16 fields=[
17 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
18 ('giturl', models.CharField(max_length=254)),
19 ('commit', models.CharField(max_length=254)),
20 ('dirpath', models.CharField(max_length=254)),
21 ],
22 ),
23 migrations.CreateModel(
24 name='BRError',
25 fields=[
26 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
27 ('errtype', models.CharField(max_length=100)),
28 ('errmsg', models.TextField()),
29 ('traceback', models.TextField()),
30 ],
31 ),
32 migrations.CreateModel(
33 name='BRLayer',
34 fields=[
35 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
36 ('name', models.CharField(max_length=100)),
37 ('giturl', models.CharField(max_length=254)),
38 ('commit', models.CharField(max_length=254)),
39 ('dirpath', models.CharField(max_length=254)),
40 ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True, on_delete=models.CASCADE)),
41 ],
42 ),
43 migrations.CreateModel(
44 name='BRTarget',
45 fields=[
46 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
47 ('target', models.CharField(max_length=100)),
48 ('task', models.CharField(max_length=100, null=True)),
49 ],
50 ),
51 migrations.CreateModel(
52 name='BRVariable',
53 fields=[
54 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
55 ('name', models.CharField(max_length=100)),
56 ('value', models.TextField(blank=True)),
57 ],
58 ),
59 migrations.CreateModel(
60 name='BuildEnvironment',
61 fields=[
62 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
63 ('address', models.CharField(max_length=254)),
64 ('betype', models.IntegerField(choices=[(0, b'local'), (1, b'ssh')])),
65 ('bbaddress', models.CharField(max_length=254, blank=True)),
66 ('bbport', models.IntegerField(default=-1)),
67 ('bbtoken', models.CharField(max_length=126, blank=True)),
68 ('bbstate', models.IntegerField(default=0, choices=[(0, b'stopped'), (1, b'started')])),
69 ('sourcedir', models.CharField(max_length=512, blank=True)),
70 ('builddir', models.CharField(max_length=512, blank=True)),
71 ('lock', models.IntegerField(default=0, choices=[(0, b'free'), (1, b'lock'), (2, b'running')])),
72 ('created', models.DateTimeField(auto_now_add=True)),
73 ('updated', models.DateTimeField(auto_now=True)),
74 ],
75 ),
76 migrations.CreateModel(
77 name='BuildRequest',
78 fields=[
79 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
80 ('state', models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'archive')])),
81 ('created', models.DateTimeField(auto_now_add=True)),
82 ('updated', models.DateTimeField(auto_now=True)),
83 ('build', models.OneToOneField(null=True, to='orm.Build', on_delete=models.CASCADE)),
84 ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True, on_delete=models.CASCADE)),
85 ('project', models.ForeignKey(to='orm.Project', on_delete=models.CASCADE)),
86 ],
87 ),
88 migrations.AddField(
89 model_name='brvariable',
90 name='req',
91 field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE),
92 ),
93 migrations.AddField(
94 model_name='brtarget',
95 name='req',
96 field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE),
97 ),
98 migrations.AddField(
99 model_name='brlayer',
100 name='req',
101 field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE),
102 ),
103 migrations.AddField(
104 model_name='brerror',
105 name='req',
106 field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE),
107 ),
108 migrations.AddField(
109 model_name='brbitbake',
110 name='req',
111 field=models.OneToOneField(to='bldcontrol.BuildRequest', on_delete=models.CASCADE),
112 ),
113 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py b/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py
deleted file mode 100644
index 0c2475aba5..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py
+++ /dev/null
@@ -1,19 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('bldcontrol', '0001_initial'),
11 ]
12
13 operations = [
14 migrations.AlterField(
15 model_name='buildenvironment',
16 name='betype',
17 field=models.IntegerField(choices=[(0, b'local')]),
18 ),
19 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py b/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py
deleted file mode 100644
index eec9216ca3..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py
+++ /dev/null
@@ -1,19 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('bldcontrol', '0002_auto_20160120_1250'),
11 ]
12
13 operations = [
14 migrations.AlterField(
15 model_name='buildrequest',
16 name='state',
17 field=models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'cancelling'), (7, b'archive')]),
18 ),
19 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py b/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py
deleted file mode 100644
index 3d90629541..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py
+++ /dev/null
@@ -1,34 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('bldcontrol', '0003_add_cancelling_state'),
11 ]
12
13 operations = [
14 migrations.AlterField(
15 model_name='buildenvironment',
16 name='bbstate',
17 field=models.IntegerField(default=0, choices=[(0, 'stopped'), (1, 'started')]),
18 ),
19 migrations.AlterField(
20 model_name='buildenvironment',
21 name='betype',
22 field=models.IntegerField(choices=[(0, 'local')]),
23 ),
24 migrations.AlterField(
25 model_name='buildenvironment',
26 name='lock',
27 field=models.IntegerField(default=0, choices=[(0, 'free'), (1, 'lock'), (2, 'running')]),
28 ),
29 migrations.AlterField(
30 model_name='buildrequest',
31 name='state',
32 field=models.IntegerField(default=0, choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'completed'), (4, 'failed'), (5, 'deleted'), (6, 'cancelling'), (7, 'archive')]),
33 ),
34 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py b/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py
deleted file mode 100644
index 4bb9517768..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py
+++ /dev/null
@@ -1,19 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('bldcontrol', '0004_auto_20160523_1446'),
11 ]
12
13 operations = [
14 migrations.AlterField(
15 model_name='buildrequest',
16 name='state',
17 field=models.IntegerField(choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'failed'), (4, 'deleted'), (5, 'cancelling'), (6, 'completed'), (7, 'archive')], default=0),
18 ),
19 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py b/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py
deleted file mode 100644
index 2460002f07..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py
+++ /dev/null
@@ -1,19 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('bldcontrol', '0005_reorder_buildrequest_states'),
11 ]
12
13 operations = [
14 migrations.AddField(
15 model_name='brlayer',
16 name='local_source_dir',
17 field=models.CharField(max_length=254, null=True),
18 ),
19 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py b/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py
deleted file mode 100644
index 4be42a4cf9..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py
+++ /dev/null
@@ -1,29 +0,0 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('bldcontrol', '0006_brlayer_local_source_dir'),
11 ]
12
13 operations = [
14 migrations.AlterField(
15 model_name='brlayer',
16 name='commit',
17 field=models.CharField(max_length=254, null=True),
18 ),
19 migrations.AlterField(
20 model_name='brlayer',
21 name='dirpath',
22 field=models.CharField(max_length=254, null=True),
23 ),
24 migrations.AlterField(
25 model_name='brlayer',
26 name='giturl',
27 field=models.CharField(max_length=254, null=True),
28 ),
29 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py
deleted file mode 100644
index 45b477d02c..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py
+++ /dev/null
@@ -1,48 +0,0 @@
1# Generated by Django 3.2.12 on 2022-03-06 03:28
2
3from django.db import migrations, models
4
5
6class Migration(migrations.Migration):
7
8 dependencies = [
9 ('bldcontrol', '0007_brlayers_optional_gitinfo'),
10 ]
11
12 operations = [
13 migrations.AlterField(
14 model_name='brbitbake',
15 name='id',
16 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
17 ),
18 migrations.AlterField(
19 model_name='brerror',
20 name='id',
21 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
22 ),
23 migrations.AlterField(
24 model_name='brlayer',
25 name='id',
26 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
27 ),
28 migrations.AlterField(
29 model_name='brtarget',
30 name='id',
31 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
32 ),
33 migrations.AlterField(
34 model_name='brvariable',
35 name='id',
36 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
37 ),
38 migrations.AlterField(
39 model_name='buildenvironment',
40 name='id',
41 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
42 ),
43 migrations.AlterField(
44 model_name='buildrequest',
45 name='id',
46 field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
47 ),
48 ]
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/__init__.py b/bitbake/lib/toaster/bldcontrol/migrations/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/bitbake/lib/toaster/bldcontrol/migrations/__init__.py
+++ /dev/null
diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py
deleted file mode 100644
index 42750e7180..0000000000
--- a/bitbake/lib/toaster/bldcontrol/models.py
+++ /dev/null
@@ -1,165 +0,0 @@
1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5from __future__ import unicode_literals
6from django.db import models
7from django.utils.encoding import force_str
8from orm.models import Project, Build, Layer_Version
9
10import logging
11logger = logging.getLogger("toaster")
12# a BuildEnvironment is the equivalent of the "build/" directory on the localhost
13class BuildEnvironment(models.Model):
14 SERVER_STOPPED = 0
15 SERVER_STARTED = 1
16 SERVER_STATE = (
17 (SERVER_STOPPED, "stopped"),
18 (SERVER_STARTED, "started"),
19 )
20
21 TYPE_LOCAL = 0
22 TYPE = (
23 (TYPE_LOCAL, "local"),
24 )
25
26 LOCK_FREE = 0
27 LOCK_LOCK = 1
28 LOCK_RUNNING = 2
29 LOCK_STATE = (
30 (LOCK_FREE, "free"),
31 (LOCK_LOCK, "lock"),
32 (LOCK_RUNNING, "running"),
33 )
34
35 address = models.CharField(max_length = 254)
36 betype = models.IntegerField(choices = TYPE)
37 bbaddress = models.CharField(max_length = 254, blank = True)
38 bbport = models.IntegerField(default = -1)
39 bbtoken = models.CharField(max_length = 126, blank = True)
40 bbstate = models.IntegerField(choices = SERVER_STATE, default = SERVER_STOPPED)
41 sourcedir = models.CharField(max_length = 512, blank = True)
42 builddir = models.CharField(max_length = 512, blank = True)
43 lock = models.IntegerField(choices = LOCK_STATE, default = LOCK_FREE)
44 created = models.DateTimeField(auto_now_add = True)
45 updated = models.DateTimeField(auto_now = True)
46
47 def get_artifact(self, path):
48 if self.betype == BuildEnvironment.TYPE_LOCAL:
49 return open(path, "r")
50 raise NotImplementedError("FIXME: artifact download not implemented "\
51 "for build environment type %s" % \
52 self.get_betype_display())
53
54 def has_artifact(self, path):
55 import os
56 if self.betype == BuildEnvironment.TYPE_LOCAL:
57 return os.path.exists(path)
58 raise NotImplementedError("FIXME: has artifact not implemented for "\
59 "build environment type %s" % \
60 self.get_betype_display())
61
62# a BuildRequest is a request that the scheduler will build using a BuildEnvironment
63# the build request queue is the table itself, ordered by state
64
65class BuildRequest(models.Model):
66 REQ_CREATED = 0
67 REQ_QUEUED = 1
68 REQ_INPROGRESS = 2
69 REQ_FAILED = 3
70 REQ_DELETED = 4
71 REQ_CANCELLING = 5
72 REQ_COMPLETED = 6
73 REQ_ARCHIVE = 7
74
75 REQUEST_STATE = (
76 (REQ_CREATED, "created"),
77 (REQ_QUEUED, "queued"),
78 (REQ_INPROGRESS, "in progress"),
79 (REQ_FAILED, "failed"),
80 (REQ_DELETED, "deleted"),
81 (REQ_CANCELLING, "cancelling"),
82 (REQ_COMPLETED, "completed"),
83 (REQ_ARCHIVE, "archive"),
84 )
85
86 search_allowed_fields = ("brtarget__target", "build__project__name")
87
88 project = models.ForeignKey(Project, on_delete=models.CASCADE)
89 build = models.OneToOneField(Build, on_delete=models.CASCADE, null = True) # TODO: toasterui should set this when Build is created
90 environment = models.ForeignKey(BuildEnvironment, on_delete=models.CASCADE, null = True)
91 state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED)
92 created = models.DateTimeField(auto_now_add = True)
93 updated = models.DateTimeField(auto_now = True)
94
95 def __init__(self, *args, **kwargs):
96 super(BuildRequest, self).__init__(*args, **kwargs)
97 # Save the old state in case it's about to be modified
98 self.old_state = self.state
99
100 def save(self, *args, **kwargs):
101 # Check that the state we're trying to set is not going backwards
102 # e.g. from REQ_FAILED to REQ_INPROGRESS
103 if self.old_state != self.state and self.old_state > self.state:
104 logger.warning("Invalid state change requested: "
105 "Cannot go from %s to %s - ignoring request" %
106 (BuildRequest.REQUEST_STATE[self.old_state][1],
107 BuildRequest.REQUEST_STATE[self.state][1])
108 )
109 # Set property back to the old value
110 self.state = self.old_state
111 return
112
113 super(BuildRequest, self).save(*args, **kwargs)
114
115
116 def get_duration(self):
117 return (self.updated - self.created).total_seconds()
118
119 def get_sorted_target_list(self):
120 tgts = self.brtarget_set.order_by( 'target' );
121 return( tgts );
122
123 def get_machine(self):
124 return self.brvariable_set.get(name="MACHINE").value
125
126 def __str__(self):
127 return force_str('%s %s' % (self.project, self.get_state_display()))
128
129# These tables specify the settings for running an actual build.
130# They MUST be kept in sync with the tables in orm.models.Project*
131
132
133class BRLayer(models.Model):
134 req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE)
135 name = models.CharField(max_length=100)
136 giturl = models.CharField(max_length=254, null=True)
137 local_source_dir = models.CharField(max_length=254, null=True)
138 commit = models.CharField(max_length=254, null=True)
139 dirpath = models.CharField(max_length=254, null=True)
140 layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True)
141
142class BRBitbake(models.Model):
143 req = models.OneToOneField(BuildRequest, on_delete=models.CASCADE) # only one bitbake for a request
144 giturl = models.CharField(max_length =254)
145 commit = models.CharField(max_length = 254)
146 dirpath = models.CharField(max_length = 254)
147
148class BRVariable(models.Model):
149 req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE)
150 name = models.CharField(max_length=100)
151 value = models.TextField(blank = True)
152
153class BRTarget(models.Model):
154 req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE)
155 target = models.CharField(max_length=100)
156 task = models.CharField(max_length=100, null=True)
157
158class BRError(models.Model):
159 req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE)
160 errtype = models.CharField(max_length=100)
161 errmsg = models.TextField()
162 traceback = models.TextField()
163
164 def __str__(self):
165 return "%s (%s)" % (self.errmsg, self.req)
diff --git a/bitbake/lib/toaster/bldcontrol/views.py b/bitbake/lib/toaster/bldcontrol/views.py
deleted file mode 100644
index 286d88b45a..0000000000
--- a/bitbake/lib/toaster/bldcontrol/views.py
+++ /dev/null
@@ -1,5 +0,0 @@
1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5# Create your views here.
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2025-11-08 05:36:46 +0000