diff options
author | Dave Cobbley <david.j.cobbley@linux.intel.com> | 2018-08-14 20:05:37 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-08-23 04:26:31 +0300 |
commit | eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 (patch) | |
tree | de291a73dc37168da6370e2cf16c347d1eba9df8 /poky/bitbake/lib/bb/ui | |
parent | 9c3cf826d853102535ead04cebc2d6023eff3032 (diff) | |
download | openbmc-eb8dc40360f0cfef56fb6947cc817a547d6d9bc6.tar.xz |
[Subtree] Removing import-layers directory
As part of the move to subtrees, need to bring all the import layers
content to the top level.
Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f
Signed-off-by: Dave Cobbley <david.j.cobbley@linux.intel.com>
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'poky/bitbake/lib/bb/ui')
34 files changed, 4166 insertions, 0 deletions
diff --git a/poky/bitbake/lib/bb/ui/__init__.py b/poky/bitbake/lib/bb/ui/__init__.py new file mode 100644 index 000000000..a4805ed02 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/__init__.py @@ -0,0 +1,17 @@ +# +# BitBake UI Implementation +# +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/poky/bitbake/lib/bb/ui/buildinfohelper.py b/poky/bitbake/lib/bb/ui/buildinfohelper.py new file mode 100644 index 000000000..524a5b094 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/buildinfohelper.py @@ -0,0 +1,2002 @@ +# +# BitBake ToasterUI Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import bb +import re +import os + +import django +from django.utils import timezone + +import toaster +# Add toaster module to the search path to help django.setup() find the right +# modules +sys.path.insert(0, os.path.dirname(toaster.__file__)) + +#Set the DJANGO_SETTINGS_MODULE if it's not already set +os.environ["DJANGO_SETTINGS_MODULE"] =\ + os.environ.get("DJANGO_SETTINGS_MODULE", + "toaster.toastermain.settings") +# Setup django framework (needs to be done before importing modules) +django.setup() + +from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText +from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile +from orm.models import Variable, VariableHistory +from orm.models import Package, Package_File, Target_Installed_Package, Target_File +from orm.models import Task_Dependency, Package_Dependency +from orm.models import Recipe_Dependency, Provides +from orm.models import Project, CustomImagePackage +from orm.models import signal_runbuilds + +from bldcontrol.models import BuildEnvironment, BuildRequest +from bldcontrol.models import BRLayer +from bldcontrol import bbcontroller + +from bb.msg import BBLogFormatter as formatter +from django.db import models +from pprint import pformat +import logging +from datetime import datetime, timedelta + +from django.db import transaction, connection + + +# pylint: disable=invalid-name +# the logger name is standard throughout BitBake +logger = logging.getLogger("ToasterLogger") + +class NotExisting(Exception): + pass + +class ORMWrapper(object): + """ This class creates the dictionaries needed to store information in the database + following the format defined by the Django models. It is also used to save this + information in the database. + """ + + def __init__(self): + self.layer_version_objects = [] + self.layer_version_built = [] + self.task_objects = {} + self.recipe_objects = {} + + @staticmethod + def _build_key(**kwargs): + key = "0" + for k in sorted(kwargs.keys()): + if isinstance(kwargs[k], models.Model): + key += "-%d" % kwargs[k].id + else: + key += "-%s" % str(kwargs[k]) + return key + + + def _cached_get_or_create(self, clazz, **kwargs): + """ This is a memory-cached get_or_create. We assume that the objects will not be created in the + database through any other means. + """ + + assert issubclass(clazz, models.Model), "_cached_get_or_create needs to get the class as first argument" + + key = ORMWrapper._build_key(**kwargs) + dictname = "objects_%s" % clazz.__name__ + if not dictname in vars(self).keys(): + vars(self)[dictname] = {} + + created = False + if not key in vars(self)[dictname].keys(): + vars(self)[dictname][key], created = \ + clazz.objects.get_or_create(**kwargs) + + return (vars(self)[dictname][key], created) + + + def _cached_get(self, clazz, **kwargs): + """ This is a memory-cached get. We assume that the objects will not change in the database between gets. + """ + assert issubclass(clazz, models.Model), "_cached_get needs to get the class as first argument" + + key = ORMWrapper._build_key(**kwargs) + dictname = "objects_%s" % clazz.__name__ + + if not dictname in vars(self).keys(): + vars(self)[dictname] = {} + + if not key in vars(self)[dictname].keys(): + vars(self)[dictname][key] = clazz.objects.get(**kwargs) + + return vars(self)[dictname][key] + + def get_similar_target_with_image_files(self, target): + """ + Get a Target object "similar" to target; i.e. with the same target + name ('core-image-minimal' etc.) and machine. + """ + return target.get_similar_target_with_image_files() + + def get_similar_target_with_sdk_files(self, target): + return target.get_similar_target_with_sdk_files() + + def clone_image_artifacts(self, target_from, target_to): + target_to.clone_image_artifacts_from(target_from) + + def clone_sdk_artifacts(self, target_from, target_to): + target_to.clone_sdk_artifacts_from(target_from) + + def _timestamp_to_datetime(self, secs): + """ + Convert timestamp in seconds to Python datetime + """ + return timezone.make_aware(datetime(1970, 1, 1) + timedelta(seconds=secs)) + + # pylint: disable=no-self-use + # we disable detection of no self use in functions because the methods actually work on the object + # even if they don't touch self anywhere + + # pylint: disable=bad-continuation + # we do not follow the python conventions for continuation indentation due to long lines here + + def get_or_create_build_object(self, brbe): + prj = None + buildrequest = None + if brbe is not None: + # Toaster-triggered build + logger.debug(1, "buildinfohelper: brbe is %s" % brbe) + br, _ = brbe.split(":") + buildrequest = BuildRequest.objects.get(pk=br) + prj = buildrequest.project + else: + # CLI build + prj = Project.objects.get_or_create_default_project() + logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) + + if buildrequest is not None: + # reuse existing Build object + build = buildrequest.build + build.project = prj + build.save() + else: + # create new Build object + now = timezone.now() + build = Build.objects.create( + project=prj, + started_on=now, + completed_on=now, + build_name='') + + logger.debug(1, "buildinfohelper: build is created %s" % build) + + if buildrequest is not None: + buildrequest.build = build + buildrequest.save() + + return build + + def update_build(self, build, data_dict): + for key in data_dict: + setattr(build, key, data_dict[key]) + build.save() + + @staticmethod + def get_or_create_targets(target_info): + """ + NB get_or_create() is used here because for Toaster-triggered builds, + we already created the targets when the build was triggered. + """ + result = [] + for target in target_info['targets']: + task = '' + if ':' in target: + target, task = target.split(':', 1) + if task.startswith('do_'): + task = task[3:] + if task == 'build': + task = '' + + obj, _ = Target.objects.get_or_create(build=target_info['build'], + target=target, + task=task) + result.append(obj) + return result + + def update_build_stats_and_outcome(self, build, errors, warnings, taskfailures): + assert isinstance(build,Build) + assert isinstance(errors, int) + assert isinstance(warnings, int) + + if build.outcome == Build.CANCELLED: + return + try: + if build.buildrequest.state == BuildRequest.REQ_CANCELLING: + return + except AttributeError: + # We may not have a buildrequest if this is a command line build + pass + + outcome = Build.SUCCEEDED + if errors or taskfailures: + outcome = Build.FAILED + + build.completed_on = timezone.now() + build.outcome = outcome + build.save() + signal_runbuilds() + + def update_target_set_license_manifest(self, target, license_manifest_path): + target.license_manifest_path = license_manifest_path + target.save() + + def update_target_set_package_manifest(self, target, package_manifest_path): + target.package_manifest_path = package_manifest_path + target.save() + + def update_task_object(self, build, task_name, recipe_name, task_stats): + """ + Find the task for build which matches the recipe and task name + to be stored + """ + task_to_update = Task.objects.get( + build = build, + task_name = task_name, + recipe__name = recipe_name + ) + + if 'started' in task_stats and 'ended' in task_stats: + task_to_update.started = self._timestamp_to_datetime(task_stats['started']) + task_to_update.ended = self._timestamp_to_datetime(task_stats['ended']) + task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started']) + task_to_update.cpu_time_user = task_stats.get('cpu_time_user') + task_to_update.cpu_time_system = task_stats.get('cpu_time_system') + if 'disk_io_read' in task_stats and 'disk_io_write' in task_stats: + task_to_update.disk_io_read = task_stats['disk_io_read'] + task_to_update.disk_io_write = task_stats['disk_io_write'] + task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write'] + + task_to_update.save() + + def get_update_task_object(self, task_information, must_exist = False): + assert 'build' in task_information + assert 'recipe' in task_information + assert 'task_name' in task_information + + # we use must_exist info for database look-up optimization + task_object, created = self._cached_get_or_create(Task, + build=task_information['build'], + recipe=task_information['recipe'], + task_name=task_information['task_name'] + ) + if created and must_exist: + task_information['debug'] = "build id %d, recipe id %d" % (task_information['build'].pk, task_information['recipe'].pk) + raise NotExisting("Task object created when expected to exist", task_information) + + object_changed = False + for v in vars(task_object): + if v in task_information.keys(): + if vars(task_object)[v] != task_information[v]: + vars(task_object)[v] = task_information[v] + object_changed = True + + # update setscene-related information if the task has a setscene + if task_object.outcome == Task.OUTCOME_COVERED and 1 == task_object.get_related_setscene().count(): + task_object.outcome = Task.OUTCOME_CACHED + object_changed = True + + outcome_task_setscene = Task.objects.get(task_executed=True, build = task_object.build, + recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").outcome + if outcome_task_setscene == Task.OUTCOME_SUCCESS: + task_object.sstate_result = Task.SSTATE_RESTORED + object_changed = True + elif outcome_task_setscene == Task.OUTCOME_FAILED: + task_object.sstate_result = Task.SSTATE_FAILED + object_changed = True + + if object_changed: + task_object.save() + return task_object + + + def get_update_recipe_object(self, recipe_information, must_exist = False): + assert 'layer_version' in recipe_information + assert 'file_path' in recipe_information + assert 'pathflags' in recipe_information + + assert not recipe_information['file_path'].startswith("/") # we should have layer-relative paths at all times + + + def update_recipe_obj(recipe_object): + object_changed = False + for v in vars(recipe_object): + if v in recipe_information.keys(): + object_changed = True + vars(recipe_object)[v] = recipe_information[v] + + if object_changed: + recipe_object.save() + + recipe, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'], + file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags']) + + update_recipe_obj(recipe) + + built_recipe = None + # Create a copy of the recipe for historical puposes and update it + for built_layer in self.layer_version_built: + if built_layer.layer == recipe_information['layer_version'].layer: + built_recipe, c = self._cached_get_or_create(Recipe, + layer_version=built_layer, + file_path=recipe_information['file_path'], + pathflags = recipe_information['pathflags']) + update_recipe_obj(built_recipe) + break + + + # If we're in analysis mode or if this is a custom recipe + # then we are wholly responsible for the data + # and therefore we return the 'real' recipe rather than the build + # history copy of the recipe. + if recipe_information['layer_version'].build is not None and \ + recipe_information['layer_version'].build.project == \ + Project.objects.get_or_create_default_project(): + return recipe + + if built_recipe is None: + return recipe + + return built_recipe + + def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information): + if isinstance(layer_obj, Layer_Version): + # We already found our layer version for this build so just + # update it with the new build information + logger.debug("We found our layer from toaster") + layer_obj.local_path = layer_version_information['local_path'] + layer_obj.save() + self.layer_version_objects.append(layer_obj) + + # create a new copy of this layer version as a snapshot for + # historical purposes + layer_copy, c = Layer_Version.objects.get_or_create( + build=build_obj, + layer=layer_obj.layer, + release=layer_obj.release, + branch=layer_version_information['branch'], + commit=layer_version_information['commit'], + local_path=layer_version_information['local_path'], + ) + + logger.debug("Created new layer version %s for build history", + layer_copy.layer.name) + + self.layer_version_built.append(layer_copy) + + return layer_obj + + assert isinstance(build_obj, Build) + assert isinstance(layer_obj, Layer) + assert 'branch' in layer_version_information + assert 'commit' in layer_version_information + assert 'priority' in layer_version_information + assert 'local_path' in layer_version_information + + # If we're doing a command line build then associate this new layer with the + # project to avoid it 'contaminating' toaster data + project = None + if build_obj.project == Project.objects.get_or_create_default_project(): + project = build_obj.project + + layer_version_object, _ = Layer_Version.objects.get_or_create( + build = build_obj, + layer = layer_obj, + branch = layer_version_information['branch'], + commit = layer_version_information['commit'], + priority = layer_version_information['priority'], + local_path = layer_version_information['local_path'], + project=project) + + self.layer_version_objects.append(layer_version_object) + + return layer_version_object + + def get_update_layer_object(self, layer_information, brbe): + assert 'name' in layer_information + assert 'layer_index_url' in layer_information + + # From command line builds we have no brbe as the request is directly + # from bitbake + if brbe is None: + # If we don't have git commit sha then we're using a non-git + # layer so set the layer_source_dir to identify it as such + if not layer_information['version']['commit']: + local_source_dir = layer_information["local_path"] + else: + local_source_dir = None + + layer_object, _ = \ + Layer.objects.get_or_create( + name=layer_information['name'], + local_source_dir=local_source_dir, + layer_index_url=layer_information['layer_index_url']) + + return layer_object + else: + br_id, be_id = brbe.split(":") + + # Find the layer version by matching the layer event information + # against the metadata we have in Toaster + + try: + br_layer = BRLayer.objects.get(req=br_id, + name=layer_information['name']) + return br_layer.layer_version + except (BRLayer.MultipleObjectsReturned, BRLayer.DoesNotExist): + # There are multiple of the same layer name or the name + # hasn't been determined by the toaster.bbclass layer + # so let's filter by the local_path + bc = bbcontroller.getBuildEnvironmentController(pk=be_id) + for br_layer in BRLayer.objects.filter(req=br_id): + if br_layer.giturl and \ + layer_information['local_path'].endswith( + bc.getGitCloneDirectory(br_layer.giturl, + br_layer.commit)): + return br_layer.layer_version + + if br_layer.local_source_dir == \ + layer_information['local_path']: + return br_layer.layer_version + + # We've reached the end of our search and couldn't find the layer + # we can continue but some data may be missing + raise NotExisting("Unidentified layer %s" % + pformat(layer_information)) + + def save_target_file_information(self, build_obj, target_obj, filedata): + assert isinstance(build_obj, Build) + assert isinstance(target_obj, Target) + dirs = filedata['dirs'] + files = filedata['files'] + syms = filedata['syms'] + + # always create the root directory as a special case; + # note that this is never displayed, so the owner, group, + # size, permission are irrelevant + tf_obj = Target_File.objects.create(target = target_obj, + path = '/', + size = 0, + owner = '', + group = '', + permission = '', + inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj.save() + + # insert directories, ordered by name depth + for d in sorted(dirs, key=lambda x:len(x[-1].split("/"))): + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + + # we already created the root directory, so ignore any + # entry for it + if len(path) == 0: + continue + + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + if len(parent_path) == 0: + parent_path = "/" + parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj = Target_File.objects.create( + target = target_obj, + path = path, + size = size, + inodetype = Target_File.ITYPE_DIRECTORY, + permission = permission, + owner = user, + group = group, + directory = parent_obj) + + + # we insert files + for d in files: + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + inodetype = Target_File.ITYPE_REGULAR + if d[0].startswith('b'): + inodetype = Target_File.ITYPE_BLOCK + if d[0].startswith('c'): + inodetype = Target_File.ITYPE_CHARACTER + if d[0].startswith('p'): + inodetype = Target_File.ITYPE_FIFO + + tf_obj = Target_File.objects.create( + target = target_obj, + path = path, + size = size, + inodetype = inodetype, + permission = permission, + owner = user, + group = group) + parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj.directory = parent_obj + tf_obj.save() + + # we insert symlinks + for d in syms: + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + filetarget_path = d[6] + + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + if not filetarget_path.startswith("/"): + # we have a relative path, get a normalized absolute one + filetarget_path = parent_path + "/" + filetarget_path + fcp = filetarget_path.split("/") + fcpl = [] + for i in fcp: + if i == "..": + fcpl.pop() + else: + fcpl.append(i) + filetarget_path = "/".join(fcpl) + + try: + filetarget_obj = Target_File.objects.get(target = target_obj, path = filetarget_path) + except Target_File.DoesNotExist: + # we might have an invalid link; no way to detect this. just set it to None + filetarget_obj = None + + parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + + tf_obj = Target_File.objects.create( + target = target_obj, + path = path, + size = size, + inodetype = Target_File.ITYPE_SYMLINK, + permission = permission, + owner = user, + group = group, + directory = parent_obj, + sym_target = filetarget_obj) + + + def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes, built_package=False): + assert isinstance(build_obj, Build) + assert isinstance(target_obj, Target) + + errormsg = "" + for p in packagedict: + # Search name swtiches round the installed name vs package name + # by default installed name == package name + searchname = p + if p not in pkgpnmap: + logger.warning("Image packages list contains %p, but is" + " missing from all packages list where the" + " metadata comes from. Skipping...", p) + continue + + if 'OPKGN' in pkgpnmap[p].keys(): + searchname = pkgpnmap[p]['OPKGN'] + + built_recipe = recipes[pkgpnmap[p]['PN']] + + if built_package: + packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = searchname ) + recipe = built_recipe + else: + packagedict[p]['object'], created = \ + CustomImagePackage.objects.get_or_create(name=searchname) + # Clear the Package_Dependency objects as we're going to update + # the CustomImagePackage with the latest dependency information + packagedict[p]['object'].package_dependencies_target.all().delete() + packagedict[p]['object'].package_dependencies_source.all().delete() + try: + recipe = self._cached_get( + Recipe, + name=built_recipe.name, + layer_version__build=None, + layer_version__release= + built_recipe.layer_version.release, + file_path=built_recipe.file_path, + version=built_recipe.version + ) + except (Recipe.DoesNotExist, + Recipe.MultipleObjectsReturned) as e: + logger.info("We did not find one recipe for the" + "configuration data package %s %s" % (p, e)) + continue + + if created or packagedict[p]['object'].size == -1: # save the data anyway we can, not just if it was not created here; bug [YOCTO #6887] + # fill in everything we can from the runtime-reverse package data + try: + packagedict[p]['object'].recipe = recipe + packagedict[p]['object'].version = pkgpnmap[p]['PV'] + packagedict[p]['object'].installed_name = p + packagedict[p]['object'].revision = pkgpnmap[p]['PR'] + packagedict[p]['object'].license = pkgpnmap[p]['LICENSE'] + packagedict[p]['object'].section = pkgpnmap[p]['SECTION'] + packagedict[p]['object'].summary = pkgpnmap[p]['SUMMARY'] + packagedict[p]['object'].description = pkgpnmap[p]['DESCRIPTION'] + packagedict[p]['object'].size = int(pkgpnmap[p]['PKGSIZE']) + + # no files recorded for this package, so save files info + packagefile_objects = [] + for targetpath in pkgpnmap[p]['FILES_INFO']: + targetfilesize = pkgpnmap[p]['FILES_INFO'][targetpath] + packagefile_objects.append(Package_File( package = packagedict[p]['object'], + path = targetpath, + size = targetfilesize)) + if len(packagefile_objects): + Package_File.objects.bulk_create(packagefile_objects) + except KeyError as e: + errormsg += " stpi: Key error, package %s key %s \n" % ( p, e ) + + # save disk installed size + packagedict[p]['object'].installed_size = packagedict[p]['size'] + packagedict[p]['object'].save() + + if built_package: + Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object']) + + packagedeps_objs = [] + for p in packagedict: + for (px,deptype) in packagedict[p]['depends']: + if deptype == 'depends': + tdeptype = Package_Dependency.TYPE_TRDEPENDS + elif deptype == 'recommends': + tdeptype = Package_Dependency.TYPE_TRECOMMENDS + + try: + packagedeps_objs.append(Package_Dependency( + package = packagedict[p]['object'], + depends_on = packagedict[px]['object'], + dep_type = tdeptype, + target = target_obj)) + except KeyError as e: + logger.warning("Could not add dependency to the package %s " + "because %s is an unknown package", p, px) + + if len(packagedeps_objs) > 0: + Package_Dependency.objects.bulk_create(packagedeps_objs) + else: + logger.info("No package dependencies created") + + if len(errormsg) > 0: + logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) + + def save_target_image_file_information(self, target_obj, file_name, file_size): + Target_Image_File.objects.create(target=target_obj, + file_name=file_name, file_size=file_size) + + def save_target_kernel_file(self, target_obj, file_name, file_size): + """ + Save kernel file (bzImage, modules*) information for a Target target_obj. + """ + TargetKernelFile.objects.create(target=target_obj, + file_name=file_name, file_size=file_size) + + def save_target_sdk_file(self, target_obj, file_name, file_size): + """ + Save SDK artifacts to the database, associating them with a + Target object. + """ + TargetSDKFile.objects.create(target=target_obj, file_name=file_name, + file_size=file_size) + + def create_logmessage(self, log_information): + assert 'build' in log_information + assert 'level' in log_information + assert 'message' in log_information + + log_object = LogMessage.objects.create( + build = log_information['build'], + level = log_information['level'], + message = log_information['message']) + + for v in vars(log_object): + if v in log_information.keys(): + vars(log_object)[v] = log_information[v] + + return log_object.save() + + + def save_build_package_information(self, build_obj, package_info, recipes, + built_package): + # assert isinstance(build_obj, Build) + + if not 'PN' in package_info.keys(): + # no package data to save (e.g. 'OPKGN'="lib64-*"|"lib32-*") + return None + + # create and save the object + pname = package_info['PKG'] + built_recipe = recipes[package_info['PN']] + if 'OPKGN' in package_info.keys(): + pname = package_info['OPKGN'] + + if built_package: + bp_object, _ = Package.objects.get_or_create( build = build_obj, + name = pname ) + recipe = built_recipe + else: + bp_object, created = \ + CustomImagePackage.objects.get_or_create(name=pname) + try: + recipe = self._cached_get(Recipe, + name=built_recipe.name, + layer_version__build=None, + file_path=built_recipe.file_path, + version=built_recipe.version) + + except (Recipe.DoesNotExist, Recipe.MultipleObjectsReturned): + logger.debug("We did not find one recipe for the configuration" + "data package %s" % pname) + return + + bp_object.installed_name = package_info['PKG'] + bp_object.recipe = recipe + bp_object.version = package_info['PKGV'] + bp_object.revision = package_info['PKGR'] + bp_object.summary = package_info['SUMMARY'] + bp_object.description = package_info['DESCRIPTION'] + bp_object.size = int(package_info['PKGSIZE']) + bp_object.section = package_info['SECTION'] + bp_object.license = package_info['LICENSE'] + bp_object.save() + + # save any attached file information + packagefile_objects = [] + for path in package_info['FILES_INFO']: + packagefile_objects.append(Package_File( package = bp_object, + path = path, + size = package_info['FILES_INFO'][path] )) + if len(packagefile_objects): + Package_File.objects.bulk_create(packagefile_objects) + + def _po_byname(p): + if built_package: + pkg, created = Package.objects.get_or_create(build=build_obj, + name=p) + else: + pkg, created = CustomImagePackage.objects.get_or_create(name=p) + + if created: + pkg.size = -1 + pkg.save() + return pkg + + packagedeps_objs = [] + # save soft dependency information + if 'RDEPENDS' in package_info and package_info['RDEPENDS']: + for p in bb.utils.explode_deps(package_info['RDEPENDS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RDEPENDS)) + if 'RPROVIDES' in package_info and package_info['RPROVIDES']: + for p in bb.utils.explode_deps(package_info['RPROVIDES']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RPROVIDES)) + if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: + for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RRECOMMENDS)) + if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: + for p in bb.utils.explode_deps(package_info['RSUGGESTS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RSUGGESTS)) + if 'RREPLACES' in package_info and package_info['RREPLACES']: + for p in bb.utils.explode_deps(package_info['RREPLACES']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RREPLACES)) + if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: + for p in bb.utils.explode_deps(package_info['RCONFLICTS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS)) + + if len(packagedeps_objs) > 0: + Package_Dependency.objects.bulk_create(packagedeps_objs) + + return bp_object + + def save_build_variables(self, build_obj, vardump): + assert isinstance(build_obj, Build) + + for k in vardump: + desc = vardump[k]['doc'] + if desc is None: + var_words = [word for word in k.split('_')] + root_var = "_".join([word for word in var_words if word.isupper()]) + if root_var and root_var != k and root_var in vardump: + desc = vardump[root_var]['doc'] + if desc is None: + desc = '' + if len(desc): + HelpText.objects.get_or_create(build=build_obj, + area=HelpText.VARIABLE, + key=k, text=desc) + if not bool(vardump[k]['func']): + value = vardump[k]['v'] + if value is None: + value = '' + variable_obj = Variable.objects.create( build = build_obj, + variable_name = k, + variable_value = value, + description = desc) + + varhist_objects = [] + for vh in vardump[k]['history']: + if not 'documentation.conf' in vh['file']: + varhist_objects.append(VariableHistory( variable = variable_obj, + file_name = vh['file'], + line_number = vh['line'], + operation = vh['op'])) + if len(varhist_objects): + VariableHistory.objects.bulk_create(varhist_objects) + + +class MockEvent(object): + """ This object is used to create event, for which normal event-processing methods can + be used, out of data that is not coming via an actual event + """ + def __init__(self): + self.msg = None + self.levelno = None + self.taskname = None + self.taskhash = None + self.pathname = None + self.lineno = None + + def getMessage(self): + """ + Simulate LogRecord message return + """ + return self.msg + + +class BuildInfoHelper(object): + """ This class gathers the build information from the server and sends it + towards the ORM wrapper for storing in the database + It is instantiated once per build + Keeps in memory all data that needs matching before writing it to the database + """ + + # tasks which produce image files; note we include '', as we set + # the task for a target to '' (i.e. 'build') if no target is + # explicitly defined + IMAGE_GENERATING_TASKS = ['', 'build', 'image', 'populate_sdk_ext'] + + # pylint: disable=protected-access + # the code will look into the protected variables of the event; no easy way around this + # pylint: disable=bad-continuation + # we do not follow the python conventions for continuation indentation due to long lines here + + def __init__(self, server, has_build_history = False, brbe = None): + self.internal_state = {} + self.internal_state['taskdata'] = {} + self.internal_state['targets'] = [] + self.task_order = 0 + self.autocommit_step = 1 + self.server = server + # we use manual transactions if the database doesn't autocommit on us + if not connection.features.autocommits_when_autocommit_is_off: + transaction.set_autocommit(False) + self.orm_wrapper = ORMWrapper() + self.has_build_history = has_build_history + self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0] + + # this is set for Toaster-triggered builds by localhostbecontroller + # via toasterui + self.brbe = brbe + + self.project = None + + logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) + + + ################### + ## methods to convert event/external info into objects that the ORM layer uses + + def _ensure_build(self): + """ + Ensure the current build object exists and is up to date with + data on the bitbake server + """ + if not 'build' in self.internal_state or not self.internal_state['build']: + # create the Build object + self.internal_state['build'] = \ + self.orm_wrapper.get_or_create_build_object(self.brbe) + + build = self.internal_state['build'] + + # update missing fields on the Build object with found data + build_info = {} + + # set to True if at least one field is going to be set + changed = False + + if not build.build_name: + build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + + # only reset the build name if the one on the server is actually + # a valid value for the build_name field + if build_name != None: + build_info['build_name'] = build_name + changed = True + + if not build.machine: + build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0] + changed = True + + if not build.distro: + build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] + changed = True + + if not build.distro_version: + build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] + changed = True + + if not build.bitbake_version: + build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] + changed = True + + if changed: + self.orm_wrapper.update_build(self.internal_state['build'], build_info) + + def _get_task_information(self, event, recipe): + assert 'taskname' in vars(event) + self._ensure_build() + + task_information = {} + task_information['build'] = self.internal_state['build'] + task_information['outcome'] = Task.OUTCOME_NA + task_information['recipe'] = recipe + task_information['task_name'] = event.taskname + try: + # some tasks don't come with a hash. and that's ok + task_information['sstate_checksum'] = event.taskhash + except AttributeError: + pass + return task_information + + def _get_layer_version_for_dependency(self, pathRE): + """ Returns the layer in the toaster db that has a full regex + match to the pathRE. pathRE - the layer path passed as a regex in the + event. It is created in cooker.py as a collection for the layer + priorities. + """ + self._ensure_build() + + def _sort_longest_path(layer_version): + assert isinstance(layer_version, Layer_Version) + return len(layer_version.local_path) + + # Our paths don't append a trailing slash + if pathRE.endswith("/"): + pathRE = pathRE[:-1] + + p = re.compile(pathRE) + path=re.sub(r'[$^]',r'',pathRE) + # Heuristics: we always match recipe to the deepest layer path in + # the discovered layers + for lvo in sorted(self.orm_wrapper.layer_version_objects, + reverse=True, key=_sort_longest_path): + if p.fullmatch(os.path.abspath(lvo.local_path)): + return lvo + if lvo.layer.local_source_dir: + if p.fullmatch(os.path.abspath(lvo.layer.local_source_dir)): + return lvo + if 0 == path.find(lvo.local_path): + # sub-layer path inside existing layer + return lvo + + # if we get here, we didn't read layers correctly; + # dump whatever information we have on the error log + logger.warning("Could not match layer dependency for path %s : %s", + pathRE, + self.orm_wrapper.layer_version_objects) + return None + + def _get_layer_version_for_path(self, path): + self._ensure_build() + + def _slkey_interactive(layer_version): + assert isinstance(layer_version, Layer_Version) + return len(layer_version.local_path) + + # Heuristics: we always match recipe to the deepest layer path in the discovered layers + for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive): + # we can match to the recipe file path + if path.startswith(lvo.local_path): + return lvo + if lvo.layer.local_source_dir and \ + path.startswith(lvo.layer.local_source_dir): + return lvo + + #if we get here, we didn't read layers correctly; dump whatever information we have on the error log + logger.warning("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects) + + #mockup the new layer + unknown_layer, _ = Layer.objects.get_or_create(name="Unidentified layer", layer_index_url="") + unknown_layer_version_obj, _ = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) + + # append it so we don't run into this error again and again + self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj) + + return unknown_layer_version_obj + + def _get_recipe_information_from_taskfile(self, taskfile): + localfilepath = taskfile.split(":")[-1] + filepath_flags = ":".join(sorted(taskfile.split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(localfilepath) + + + + recipe_info = {} + recipe_info['layer_version'] = layer_version_obj + recipe_info['file_path'] = localfilepath + recipe_info['pathflags'] = filepath_flags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + + return recipe_info + + def _get_path_information(self, task_object): + self._ensure_build() + + assert isinstance(task_object, Task) + build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" + build_stats_path = [] + + for t in self.internal_state['targets']: + buildname = self.internal_state['build'].build_name + pe, pv = task_object.recipe.version.split(":",1) + if len(pe) > 0: + package = task_object.recipe.name + "-" + pe + "_" + pv + else: + package = task_object.recipe.name + "-" + pv + + build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, + buildname=buildname, + package=package)) + + return build_stats_path + + + ################################ + ## external available methods to store information + @staticmethod + def _get_data_from_event(event): + evdata = None + if '_localdata' in vars(event): + evdata = event._localdata + elif 'data' in vars(event): + evdata = event.data + else: + raise Exception("Event with neither _localdata or data properties") + return evdata + + def store_layer_info(self, event): + layerinfos = BuildInfoHelper._get_data_from_event(event) + self.internal_state['lvs'] = {} + for layer in layerinfos: + try: + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] + except NotExisting as nee: + logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) + + def store_started_build(self): + self._ensure_build() + + def save_build_log_file_path(self, build_log_path): + self._ensure_build() + + if not self.internal_state['build'].cooker_log_path: + data_dict = {'cooker_log_path': build_log_path} + self.orm_wrapper.update_build(self.internal_state['build'], data_dict) + + def save_build_targets(self, event): + self._ensure_build() + + # create target information + assert '_pkgs' in vars(event) + target_information = {} + target_information['targets'] = event._pkgs + target_information['build'] = self.internal_state['build'] + + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) + + def save_build_layers_and_variables(self): + self._ensure_build() + + build_obj = self.internal_state['build'] + + # save layer version information for this build + if not 'lvs' in self.internal_state: + logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.") + else: + for layer_obj in self.internal_state['lvs']: + self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) + + del self.internal_state['lvs'] + + # Save build configuration + data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] + + # convert the paths from absolute to relative to either the build directory or layer checkouts + path_prefixes = [] + + if self.brbe is not None: + _, be_id = self.brbe.split(":") + be = BuildEnvironment.objects.get(pk = be_id) + path_prefixes.append(be.builddir) + + for layer in sorted(self.orm_wrapper.layer_version_objects, key = lambda x:len(x.local_path), reverse=True): + path_prefixes.append(layer.local_path) + + # we strip the prefixes + for k in data: + if not bool(data[k]['func']): + for vh in data[k]['history']: + if not 'documentation.conf' in vh['file']: + abs_file_name = vh['file'] + for pp in path_prefixes: + if abs_file_name.startswith(pp + "/"): + # preserve layer name in relative path + vh['file']=abs_file_name[pp.rfind("/")+1:] + break + + # save the variables + self.orm_wrapper.save_build_variables(build_obj, data) + + return self.brbe + + def set_recipes_to_parse(self, num_recipes): + """ + Set the number of recipes which need to be parsed for this build. + This is set the first time ParseStarted is received by toasterui. + """ + self._ensure_build() + self.internal_state['build'].recipes_to_parse = num_recipes + self.internal_state['build'].save() + + def set_recipes_parsed(self, num_recipes): + """ + Set the number of recipes parsed so far for this build; this is updated + each time a ParseProgress or ParseCompleted event is received by + toasterui. + """ + self._ensure_build() + if num_recipes <= self.internal_state['build'].recipes_to_parse: + self.internal_state['build'].recipes_parsed = num_recipes + self.internal_state['build'].save() + + def update_target_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + + for t in self.internal_state['targets']: + if t.is_image == True: + output_files = list(evdata.keys()) + for output in output_files: + if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): + self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) + + def update_artifact_image_file(self, event): + self._ensure_build() + evdata = BuildInfoHelper._get_data_from_event(event) + for artifact_path in evdata.keys(): + self.orm_wrapper.save_artifact_information( + self.internal_state['build'], artifact_path, + evdata[artifact_path]) + + def update_build_information(self, event, errors, warnings, taskfailures): + self._ensure_build() + self.orm_wrapper.update_build_stats_and_outcome( + self.internal_state['build'], errors, warnings, taskfailures) + + def store_started_task(self, event): + assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) + assert 'taskfile' in vars(event) + localfilepath = event.taskfile.split(":")[-1] + assert localfilepath.startswith("/") + + identifier = event.taskfile + ":" + event.taskname + + recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) + + task_information = self._get_task_information(event, recipe) + task_information['outcome'] = Task.OUTCOME_NA + + if isinstance(event, bb.runqueue.runQueueTaskSkipped): + assert 'reason' in vars(event) + task_information['task_executed'] = False + if event.reason == "covered": + task_information['outcome'] = Task.OUTCOME_COVERED + if event.reason == "existing": + task_information['outcome'] = Task.OUTCOME_PREBUILT + else: + task_information['task_executed'] = True + if 'noexec' in vars(event) and event.noexec == True: + task_information['task_executed'] = False + task_information['outcome'] = Task.OUTCOME_EMPTY + task_information['script_type'] = Task.CODING_NA + + # do not assign order numbers to scene tasks + if not isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.task_order += 1 + task_information['order'] = self.task_order + + self.orm_wrapper.get_update_task_object(task_information) + + self.internal_state['taskdata'][identifier] = { + 'outcome': task_information['outcome'], + } + + + def store_tasks_stats(self, event): + self._ensure_build() + task_data = BuildInfoHelper._get_data_from_event(event) + + for (task_file, task_name, task_stats, recipe_name) in task_data: + build = self.internal_state['build'] + self.orm_wrapper.update_task_object(build, task_name, recipe_name, task_stats) + + def update_and_store_task(self, event): + assert 'taskfile' in vars(event) + localfilepath = event.taskfile.split(":")[-1] + assert localfilepath.startswith("/") + + identifier = event.taskfile + ":" + event.taskname + if not identifier in self.internal_state['taskdata']: + if isinstance(event, bb.build.TaskBase): + # we do a bit of guessing + candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)] + if len(candidates) == 1: + identifier = candidates[0] + elif len(candidates) > 1 and hasattr(event,'_package'): + if 'native-' in event._package: + identifier = 'native:' + identifier + if 'nativesdk-' in event._package: + identifier = 'nativesdk:' + identifier + candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)] + if len(candidates) == 1: + identifier = candidates[0] + + assert identifier in self.internal_state['taskdata'] + identifierlist = identifier.split(":") + realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1]) + recipe_information = self._get_recipe_information_from_taskfile(realtaskfile) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) + task_information = self._get_task_information(event,recipe) + + task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome'] + + if 'logfile' in vars(event): + task_information['logfile'] = event.logfile + + if '_message' in vars(event): + task_information['message'] = event._message + + if 'taskflags' in vars(event): + # with TaskStarted, we get even more information + if 'python' in event.taskflags.keys() and event.taskflags['python'] == '1': + task_information['script_type'] = Task.CODING_PYTHON + else: + task_information['script_type'] = Task.CODING_SHELL + + if task_information['outcome'] == Task.OUTCOME_NA: + if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)): + task_information['outcome'] = Task.OUTCOME_SUCCESS + del self.internal_state['taskdata'][identifier] + + if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runqueue.sceneQueueTaskFailed)): + task_information['outcome'] = Task.OUTCOME_FAILED + del self.internal_state['taskdata'][identifier] + + if not connection.features.autocommits_when_autocommit_is_off: + # we force a sync point here, to get the progress bar to show + if self.autocommit_step % 3 == 0: + transaction.set_autocommit(True) + transaction.set_autocommit(False) + self.autocommit_step += 1 + + self.orm_wrapper.get_update_task_object(task_information, True) # must exist + + + def store_missed_state_tasks(self, event): + for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['missed']: + + # identifier = fn + taskname + "_setscene" + recipe_information = self._get_recipe_information_from_taskfile(fn) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + mevent = MockEvent() + mevent.taskname = taskname + mevent.taskhash = taskhash + task_information = self._get_task_information(mevent,recipe) + + task_information['start_time'] = timezone.now() + task_information['outcome'] = Task.OUTCOME_NA + task_information['sstate_checksum'] = taskhash + task_information['sstate_result'] = Task.SSTATE_MISS + task_information['path_to_sstate_obj'] = sstatefile + + self.orm_wrapper.get_update_task_object(task_information) + + for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['found']: + + # identifier = fn + taskname + "_setscene" + recipe_information = self._get_recipe_information_from_taskfile(fn) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + mevent = MockEvent() + mevent.taskname = taskname + mevent.taskhash = taskhash + task_information = self._get_task_information(mevent,recipe) + + task_information['path_to_sstate_obj'] = sstatefile + + self.orm_wrapper.get_update_task_object(task_information) + + + def store_target_package_data(self, event): + self._ensure_build() + + # for all image targets + for target in self.internal_state['targets']: + if target.is_image: + pkgdata = BuildInfoHelper._get_data_from_event(event)['pkgdata'] + imgdata = BuildInfoHelper._get_data_from_event(event)['imgdata'].get(target.target, {}) + filedata = BuildInfoHelper._get_data_from_event(event)['filedata'].get(target.target, {}) + + try: + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'], built_package=True) + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata.copy(), pkgdata, self.internal_state['recipes'], built_package=False) + except KeyError as e: + logger.warning("KeyError in save_target_package_information" + "%s ", e) + + # only try to find files in the image if the task for this + # target is one which produces image files; otherwise, the old + # list of files in the files-in-image.txt file will be + # appended to the target even if it didn't produce any images + if target.task in BuildInfoHelper.IMAGE_GENERATING_TASKS: + try: + self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata) + except KeyError as e: + logger.warning("KeyError in save_target_file_information" + "%s ", e) + + + + def cancel_cli_build(self): + """ + If a build is currently underway, set its state to CANCELLED; + note that this only gets called for command line builds which are + interrupted, so it doesn't touch any BuildRequest objects + """ + self._ensure_build() + self.internal_state['build'].outcome = Build.CANCELLED + self.internal_state['build'].save() + signal_runbuilds() + + def store_dependency_information(self, event): + assert '_depgraph' in vars(event) + assert 'layer-priorities' in event._depgraph + assert 'pn' in event._depgraph + assert 'tdepends' in event._depgraph + + errormsg = "" + + # save layer version priorities + if 'layer-priorities' in event._depgraph.keys(): + for lv in event._depgraph['layer-priorities']: + (_, path, _, priority) = lv + layer_version_obj = self._get_layer_version_for_dependency(path) + if layer_version_obj: + layer_version_obj.priority = priority + layer_version_obj.save() + + # save recipe information + self.internal_state['recipes'] = {} + for pn in event._depgraph['pn']: + + file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1] + pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(file_name) + + assert layer_version_obj is not None + + recipe_info = {} + recipe_info['name'] = pn + recipe_info['layer_version'] = layer_version_obj + + if 'version' in event._depgraph['pn'][pn]: + recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":") + + if 'summary' in event._depgraph['pn'][pn]: + recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] + + if 'license' in event._depgraph['pn'][pn]: + recipe_info['license'] = event._depgraph['pn'][pn]['license'] + + if 'description' in event._depgraph['pn'][pn]: + recipe_info['description'] = event._depgraph['pn'][pn]['description'] + + if 'section' in event._depgraph['pn'][pn]: + recipe_info['section'] = event._depgraph['pn'][pn]['section'] + + if 'homepage' in event._depgraph['pn'][pn]: + recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] + + if 'bugtracker' in event._depgraph['pn'][pn]: + recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] + + recipe_info['file_path'] = file_name + recipe_info['pathflags'] = pathflags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + + recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) + recipe.is_image = False + if 'inherits' in event._depgraph['pn'][pn].keys(): + for cls in event._depgraph['pn'][pn]['inherits']: + if cls.endswith('/image.bbclass'): + recipe.is_image = True + recipe_info['is_image'] = True + # Save the is_image state to the relevant recipe objects + self.orm_wrapper.get_update_recipe_object(recipe_info) + break + if recipe.is_image: + for t in self.internal_state['targets']: + if pn == t.target: + t.is_image = True + t.save() + self.internal_state['recipes'][pn] = recipe + + # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED + + assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split() + + # save recipe dependency + # buildtime + recipedeps_objects = [] + for recipe in event._depgraph['depends']: + target = self.internal_state['recipes'][recipe] + for dep in event._depgraph['depends'][recipe]: + if dep in assume_provided: + continue + via = None + if 'providermap' in event._depgraph and dep in event._depgraph['providermap']: + deprecipe = event._depgraph['providermap'][dep][0] + dependency = self.internal_state['recipes'][deprecipe] + via = Provides.objects.get_or_create(name=dep, + recipe=dependency)[0] + elif dep in self.internal_state['recipes']: + dependency = self.internal_state['recipes'][dep] + else: + errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep) + continue + recipe_dep = Recipe_Dependency(recipe=target, + depends_on=dependency, + via=via, + dep_type=Recipe_Dependency.TYPE_DEPENDS) + recipedeps_objects.append(recipe_dep) + + Recipe_Dependency.objects.bulk_create(recipedeps_objects) + + # save all task information + def _save_a_task(taskdesc): + spec = re.split(r'\.', taskdesc) + pn = ".".join(spec[0:-1]) + taskname = spec[-1] + e = event + e.taskname = pn + recipe = self.internal_state['recipes'][pn] + task_info = self._get_task_information(e, recipe) + task_info['task_name'] = taskname + task_obj = self.orm_wrapper.get_update_task_object(task_info) + return task_obj + + # create tasks + tasks = {} + for taskdesc in event._depgraph['tdepends']: + tasks[taskdesc] = _save_a_task(taskdesc) + + # create dependencies between tasks + taskdeps_objects = [] + for taskdesc in event._depgraph['tdepends']: + target = tasks[taskdesc] + for taskdep in event._depgraph['tdepends'][taskdesc]: + if taskdep not in tasks: + # Fetch tasks info is not collected previously + dep = _save_a_task(taskdep) + else: + dep = tasks[taskdep] + taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) + Task_Dependency.objects.bulk_create(taskdeps_objects) + + if len(errormsg) > 0: + logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", errormsg) + + + def store_build_package_information(self, event): + self._ensure_build() + + package_info = BuildInfoHelper._get_data_from_event(event) + self.orm_wrapper.save_build_package_information( + self.internal_state['build'], + package_info, + self.internal_state['recipes'], + built_package=True) + + self.orm_wrapper.save_build_package_information( + self.internal_state['build'], + package_info, + self.internal_state['recipes'], + built_package=False) + + def _store_build_done(self, errorcode): + logger.info("Build exited with errorcode %d", errorcode) + + if not self.brbe: + return + + br_id, be_id = self.brbe.split(":") + + br = BuildRequest.objects.get(pk = br_id) + + # if we're 'done' because we got cancelled update the build outcome + if br.state == BuildRequest.REQ_CANCELLING: + logger.info("Build cancelled") + br.build.outcome = Build.CANCELLED + br.build.save() + self.internal_state['build'] = br.build + errorcode = 0 + + if errorcode == 0: + # request archival of the project artifacts + br.state = BuildRequest.REQ_COMPLETED + else: + br.state = BuildRequest.REQ_FAILED + br.save() + + be = BuildEnvironment.objects.get(pk = be_id) + be.lock = BuildEnvironment.LOCK_FREE + be.save() + signal_runbuilds() + + def store_log_error(self, text): + mockevent = MockEvent() + mockevent.levelno = formatter.ERROR + mockevent.msg = text + mockevent.pathname = '-- None' + mockevent.lineno = LogMessage.ERROR + self.store_log_event(mockevent) + + def store_log_exception(self, text, backtrace = ""): + mockevent = MockEvent() + mockevent.levelno = -1 + mockevent.msg = text + mockevent.pathname = backtrace + mockevent.lineno = -1 + self.store_log_event(mockevent) + + def store_log_event(self, event): + self._ensure_build() + + if event.levelno < formatter.WARNING: + return + + # early return for CLI builds + if self.brbe is None: + if not 'backlog' in self.internal_state: + self.internal_state['backlog'] = [] + self.internal_state['backlog'].append(event) + return + + if 'backlog' in self.internal_state: + # if we have a backlog of events, do our best to save them here + if len(self.internal_state['backlog']): + tempevent = self.internal_state['backlog'].pop() + logger.debug(1, "buildinfohelper: Saving stored event %s " + % tempevent) + self.store_log_event(tempevent) + else: + logger.info("buildinfohelper: All events saved") + del self.internal_state['backlog'] + + log_information = {} + log_information['build'] = self.internal_state['build'] + if event.levelno == formatter.CRITICAL: + log_information['level'] = LogMessage.CRITICAL + elif event.levelno == formatter.ERROR: + log_information['level'] = LogMessage.ERROR + elif event.levelno == formatter.WARNING: + log_information['level'] = LogMessage.WARNING + elif event.levelno == -2: # toaster self-logging + log_information['level'] = -2 + else: + log_information['level'] = LogMessage.INFO + + log_information['message'] = event.getMessage() + log_information['pathname'] = event.pathname + log_information['lineno'] = event.lineno + logger.info("Logging error 2: %s", log_information) + + self.orm_wrapper.create_logmessage(log_information) + + def _get_filenames_from_image_license(self, image_license_manifest_path): + """ + Find the FILES line in the image_license.manifest file, + which has the basenames of the bzImage and modules files + in this format: + FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz + """ + files = [] + with open(image_license_manifest_path) as image_license: + for line in image_license: + if line.startswith('FILES'): + files_str = line.split(':')[1].strip() + files_str = re.sub(r' {2,}', ' ', files_str) + + # ignore lines like "FILES:" with no filenames + if files_str: + files += files_str.split(' ') + return files + + def _endswith(self, str_to_test, endings): + """ + Returns True if str ends with one of the strings in the list + endings, False otherwise + """ + endswith = False + for ending in endings: + if str_to_test.endswith(ending): + endswith = True + break + return endswith + + def scan_task_artifacts(self, event): + """ + The 'TaskArtifacts' event passes the manifest file content for the + tasks 'do_deploy', 'do_image_complete', 'do_populate_sdk', and + 'do_populate_sdk_ext'. The first two will be implemented later. + """ + task_vars = BuildInfoHelper._get_data_from_event(event) + task_name = task_vars['task'][task_vars['task'].find(':')+1:] + task_artifacts = task_vars['artifacts'] + + if task_name in ['do_populate_sdk', 'do_populate_sdk_ext']: + targets = [target for target in self.internal_state['targets'] \ + if target.task == task_name[3:]] + if not targets: + logger.warning("scan_task_artifacts: SDK targets not found: %s\n", task_name) + return + for artifact_path in task_artifacts: + if not os.path.isfile(artifact_path): + logger.warning("scan_task_artifacts: artifact file not found: %s\n", artifact_path) + continue + for target in targets: + # don't record the file if it's already been added + # to this target + matching_files = TargetSDKFile.objects.filter( + target=target, file_name=artifact_path) + if matching_files.count() == 0: + artifact_size = os.stat(artifact_path).st_size + self.orm_wrapper.save_target_sdk_file( + target, artifact_path, artifact_size) + + def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions): + """ + Find files in deploy_dir_image whose basename starts with the + string image_name and ends with one of the strings in + image_file_extensions. + + Returns a list of file dictionaries like + + [ + { + 'path': '/path/to/image/file', + 'size': <file size in bytes> + } + ] + """ + image_files = [] + + for dirpath, _, filenames in os.walk(deploy_dir_image): + for filename in filenames: + if filename.startswith(image_name) and \ + self._endswith(filename, image_file_extensions): + image_file_path = os.path.join(dirpath, filename) + image_file_size = os.stat(image_file_path).st_size + + image_files.append({ + 'path': image_file_path, + 'size': image_file_size + }) + + return image_files + + def scan_image_artifacts(self): + """ + Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them + with a Target object in self.internal_state['targets']. + + We have two situations to handle: + + 1. This is the first time a target + machine has been built, so + add files from the DEPLOY_DIR_IMAGE to the target. + + OR + + 2. There are no new files for the target (they were already produced by + a previous build), so copy them from the most recent previous build with + the same target, task and machine. + """ + deploy_dir_image = \ + self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0] + + # if there's no DEPLOY_DIR_IMAGE, there aren't going to be + # any image artifacts, so we can return immediately + if not deploy_dir_image: + return + + buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0] + machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] + image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] + + # location of the manifest files for this build; + # note that this file is only produced if an image is produced + license_directory = \ + self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0] + + # file name extensions for image files + image_file_extensions_unique = {} + image_fstypes = self.server.runCommand( + ['getVariable', 'IMAGE_FSTYPES'])[0] + if image_fstypes != None: + image_types_str = image_fstypes.strip() + image_file_extensions = re.sub(r' {2,}', ' ', image_types_str) + image_file_extensions_unique = set(image_file_extensions.split(' ')) + + targets = self.internal_state['targets'] + + # filter out anything which isn't an image target + image_targets = [target for target in targets if target.is_image] + + for image_target in image_targets: + # this is set to True if we find at least one file relating to + # this target; if this remains False after the scan, we copy the + # files from the most-recent Target with the same target + machine + # onto this Target instead + has_files = False + + # we construct this because by the time we reach + # BuildCompleted, this has reset to + # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; + # we need to change it to + # <TARGET>-<MACHINE>-<BUILDNAME> + real_image_name = re.sub(r'^defaultpkgname', image_target.target, + image_name) + + image_license_manifest_path = os.path.join( + license_directory, + real_image_name, + 'image_license.manifest') + + image_package_manifest_path = os.path.join( + license_directory, + real_image_name, + 'image_license.manifest') + + # if image_license.manifest exists, we can read the names of + # bzImage, modules etc. files for this build from it, then look for + # them in the DEPLOY_DIR_IMAGE; note that this file is only produced + # if an image file was produced + if os.path.isfile(image_license_manifest_path): + has_files = True + + basenames = self._get_filenames_from_image_license( + image_license_manifest_path) + + for basename in basenames: + artifact_path = os.path.join(deploy_dir_image, basename) + if not os.path.exists(artifact_path): + logger.warning("artifact %s doesn't exist, skipping" % artifact_path) + continue + artifact_size = os.stat(artifact_path).st_size + + # note that the artifact will only be saved against this + # build if it hasn't been already + self.orm_wrapper.save_target_kernel_file(image_target, + artifact_path, artifact_size) + + # store the license manifest path on the target + # (this file is also created any time an image file is created) + license_manifest_path = os.path.join(license_directory, + real_image_name, 'license.manifest') + + self.orm_wrapper.update_target_set_license_manifest( + image_target, license_manifest_path) + + # store the package manifest path on the target (this file + # is created any time an image file is created) + package_manifest_path = os.path.join(deploy_dir_image, + real_image_name + '.rootfs.manifest') + + if os.path.exists(package_manifest_path): + self.orm_wrapper.update_target_set_package_manifest( + image_target, package_manifest_path) + + # scan the directory for image files relating to this build + # (via real_image_name); note that we don't have to set + # has_files = True, as searching for the license manifest file + # will already have set it to true if at least one image file was + # produced; note that the real_image_name includes BUILDNAME, which + # in turn includes a timestamp; so if no files were produced for + # this timestamp (i.e. the build reused existing image files already + # in the directory), no files will be recorded against this target + image_files = self._get_image_files(deploy_dir_image, + real_image_name, image_file_extensions_unique) + + for image_file in image_files: + self.orm_wrapper.save_target_image_file_information( + image_target, image_file['path'], image_file['size']) + + if not has_files: + # copy image files and build artifacts from the + # most-recently-built Target with the + # same target + machine as this Target; also copy the license + # manifest path, as that is not treated as an artifact and needs + # to be set separately + similar_target = \ + self.orm_wrapper.get_similar_target_with_image_files( + image_target) + + if similar_target: + logger.info('image artifacts for target %s cloned from ' \ + 'target %s' % (image_target.pk, similar_target.pk)) + self.orm_wrapper.clone_image_artifacts(similar_target, + image_target) + + def _get_sdk_targets(self): + """ + Return targets which could generate SDK artifacts, i.e. + "do_populate_sdk" and "do_populate_sdk_ext". + """ + return [target for target in self.internal_state['targets'] \ + if target.task in ['populate_sdk', 'populate_sdk_ext']] + + def scan_sdk_artifacts(self, event): + """ + Note that we have to intercept an SDKArtifactInfo event from + toaster.bbclass (via toasterui) to get hold of the SDK variables we + need to be able to scan for files accurately: this is because + variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time + BuildCompleted is fired by bitbake, so we have to get those values + while the build is still in progress. + + For populate_sdk_ext, this runs twice, with two different + TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the + files in the SDK output directory. + """ + sdk_vars = BuildInfoHelper._get_data_from_event(event) + toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME'] + + # targets which might have created SDK artifacts + sdk_targets = self._get_sdk_targets() + + # location of SDK artifacts + tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0] + sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk') + + # all files in the SDK directory + artifacts = [] + for dir_path, _, filenames in os.walk(sdk_dir): + for filename in filenames: + full_path = os.path.join(dir_path, filename) + if not os.path.islink(full_path): + artifacts.append(full_path) + + for sdk_target in sdk_targets: + # find files in the SDK directory which haven't already been + # recorded against a Target and whose basename matches + # TOOLCHAIN_OUTPUTNAME + for artifact_path in artifacts: + basename = os.path.basename(artifact_path) + + toolchain_match = basename.startswith(toolchain_outputname) + + # files which match the name of the target which produced them; + # for example, + # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh + target_match = re.search(sdk_target.target, basename) + + # targets which produce "*-nativesdk-*" files + is_ext_sdk_target = sdk_target.task in \ + ['do_populate_sdk_ext', 'populate_sdk_ext'] + + # SDK files which don't match the target name, i.e. + # x86_64-nativesdk-libc.* + # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot* + is_ext_sdk_file = re.search('-nativesdk-', basename) + + file_from_target = (toolchain_match and target_match) or \ + (is_ext_sdk_target and is_ext_sdk_file) + + if file_from_target: + # don't record the file if it's already been added to this + # target + matching_files = TargetSDKFile.objects.filter( + target=sdk_target, file_name=artifact_path) + + if matching_files.count() == 0: + artifact_size = os.stat(artifact_path).st_size + + self.orm_wrapper.save_target_sdk_file( + sdk_target, artifact_path, artifact_size) + + def clone_required_sdk_artifacts(self): + """ + If an SDK target doesn't have any SDK artifacts, this means that + the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which + in turn means that the targets of this build didn't generate any new + artifacts. + + In this case, clone SDK artifacts for targets in the current build + from existing targets for this build. + """ + sdk_targets = self._get_sdk_targets() + for sdk_target in sdk_targets: + # only clone for SDK targets which have no TargetSDKFiles yet + if sdk_target.targetsdkfile_set.all().count() == 0: + similar_target = \ + self.orm_wrapper.get_similar_target_with_sdk_files( + sdk_target) + if similar_target: + logger.info('SDK artifacts for target %s cloned from ' \ + 'target %s' % (sdk_target.pk, similar_target.pk)) + self.orm_wrapper.clone_sdk_artifacts(similar_target, + sdk_target) + + def close(self, errorcode): + self._store_build_done(errorcode) + + if 'backlog' in self.internal_state: + # we save missed events in the database for the current build + tempevent = self.internal_state['backlog'].pop() + self.store_log_event(tempevent) + + if not connection.features.autocommits_when_autocommit_is_off: + transaction.set_autocommit(True) + + # unset the brbe; this is to prevent subsequent command-line builds + # being incorrectly attached to the previous Toaster-triggered build; + # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021 + self.brbe = None + + # unset the internal Build object to prevent it being reused for the + # next build + self.internal_state['build'] = None diff --git a/poky/bitbake/lib/bb/ui/icons/images/images_display.png b/poky/bitbake/lib/bb/ui/icons/images/images_display.png Binary files differnew file mode 100644 index 000000000..a7f87101a --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/images/images_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/images/images_hover.png b/poky/bitbake/lib/bb/ui/icons/images/images_hover.png Binary files differnew file mode 100644 index 000000000..2d9cd99b8 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/images/images_hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png b/poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png Binary files differnew file mode 100644 index 000000000..526df770d --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/add-hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/add.png b/poky/bitbake/lib/bb/ui/icons/indicators/add.png Binary files differnew file mode 100644 index 000000000..31e7090d6 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/add.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/alert.png b/poky/bitbake/lib/bb/ui/icons/indicators/alert.png Binary files differnew file mode 100644 index 000000000..d1c6f55a2 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/alert.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png b/poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png Binary files differnew file mode 100644 index 000000000..3a5402d1e --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/confirmation.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/denied.png b/poky/bitbake/lib/bb/ui/icons/indicators/denied.png Binary files differnew file mode 100644 index 000000000..ee35c7def --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/denied.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/error.png b/poky/bitbake/lib/bb/ui/icons/indicators/error.png Binary files differnew file mode 100644 index 000000000..d06a8c151 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/error.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/info.png b/poky/bitbake/lib/bb/ui/icons/indicators/info.png Binary files differnew file mode 100644 index 000000000..ee8e8d846 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/info.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/issues.png b/poky/bitbake/lib/bb/ui/icons/indicators/issues.png Binary files differnew file mode 100644 index 000000000..b0c746133 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/issues.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/refresh.png b/poky/bitbake/lib/bb/ui/icons/indicators/refresh.png Binary files differnew file mode 100644 index 000000000..eb6c419db --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/refresh.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png b/poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png Binary files differnew file mode 100644 index 000000000..aa57c6998 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/remove-hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/remove.png b/poky/bitbake/lib/bb/ui/icons/indicators/remove.png Binary files differnew file mode 100644 index 000000000..05c3c293d --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/remove.png diff --git a/poky/bitbake/lib/bb/ui/icons/indicators/tick.png b/poky/bitbake/lib/bb/ui/icons/indicators/tick.png Binary files differnew file mode 100644 index 000000000..beaad361c --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/indicators/tick.png diff --git a/poky/bitbake/lib/bb/ui/icons/info/info_display.png b/poky/bitbake/lib/bb/ui/icons/info/info_display.png Binary files differnew file mode 100644 index 000000000..5afbba29f --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/info/info_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/info/info_hover.png b/poky/bitbake/lib/bb/ui/icons/info/info_hover.png Binary files differnew file mode 100644 index 000000000..f9d294dfa --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/info/info_hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/layers/layers_display.png b/poky/bitbake/lib/bb/ui/icons/layers/layers_display.png Binary files differnew file mode 100644 index 000000000..b7f9053a9 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/layers/layers_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png b/poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png Binary files differnew file mode 100644 index 000000000..0bf3ce0db --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/layers/layers_hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/packages/packages_display.png b/poky/bitbake/lib/bb/ui/icons/packages/packages_display.png Binary files differnew file mode 100644 index 000000000..f5d0a5064 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/packages/packages_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png b/poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png Binary files differnew file mode 100644 index 000000000..c081165f3 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/packages/packages_hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png b/poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png Binary files differnew file mode 100644 index 000000000..e9809bc7d --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/recipe/recipe_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png b/poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png Binary files differnew file mode 100644 index 000000000..7e48da9af --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/recipe/recipe_hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/settings/settings_display.png b/poky/bitbake/lib/bb/ui/icons/settings/settings_display.png Binary files differnew file mode 100644 index 000000000..88c464db0 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/settings/settings_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png b/poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png Binary files differnew file mode 100644 index 000000000..d92a0bf2c --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/settings/settings_hover.png diff --git a/poky/bitbake/lib/bb/ui/icons/templates/templates_display.png b/poky/bitbake/lib/bb/ui/icons/templates/templates_display.png Binary files differnew file mode 100644 index 000000000..153c7afb6 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/templates/templates_display.png diff --git a/poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png b/poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png Binary files differnew file mode 100644 index 000000000..afb7165fe --- /dev/null +++ b/poky/bitbake/lib/bb/ui/icons/templates/templates_hover.png diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py new file mode 100644 index 000000000..fa88e6ccd --- /dev/null +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -0,0 +1,728 @@ +# +# BitBake (No)TTY UI Implementation +# +# Handling output to TTYs or files (no TTY) +# +# Copyright (C) 2006-2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import division + +import os +import sys +import xmlrpc.client as xmlrpclib +import logging +import progressbar +import signal +import bb.msg +import time +import fcntl +import struct +import copy +import atexit + +from bb.ui import uihelper + +featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS] + +logger = logging.getLogger("BitBake") +interactive = sys.stdout.isatty() + +class BBProgress(progressbar.ProgressBar): + def __init__(self, msg, maxval, widgets=None, extrapos=-1, resize_handler=None): + self.msg = msg + self.extrapos = extrapos + if not widgets: + widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ', + progressbar.ETA()] + self.extrapos = 4 + + if resize_handler: + self._resize_default = resize_handler + else: + self._resize_default = signal.getsignal(signal.SIGWINCH) + progressbar.ProgressBar.__init__(self, maxval, [self.msg + ": "] + widgets, fd=sys.stdout) + + def _handle_resize(self, signum=None, frame=None): + progressbar.ProgressBar._handle_resize(self, signum, frame) + if self._resize_default: + self._resize_default(signum, frame) + + def finish(self): + progressbar.ProgressBar.finish(self) + if self._resize_default: + signal.signal(signal.SIGWINCH, self._resize_default) + + def setmessage(self, msg): + self.msg = msg + self.widgets[0] = msg + + def setextra(self, extra): + if self.extrapos > -1: + if extra: + extrastr = str(extra) + if extrastr[0] != ' ': + extrastr = ' ' + extrastr + else: + extrastr = '' + self.widgets[self.extrapos] = extrastr + + def _need_update(self): + # We always want the bar to print when update() is called + return True + +class NonInteractiveProgress(object): + fobj = sys.stdout + + def __init__(self, msg, maxval): + self.msg = msg + self.maxval = maxval + self.finished = False + + def start(self, update=True): + self.fobj.write("%s..." % self.msg) + self.fobj.flush() + return self + + def update(self, value): + pass + + def finish(self): + if self.finished: + return + self.fobj.write("done.\n") + self.fobj.flush() + self.finished = True + +def new_progress(msg, maxval): + if interactive: + return BBProgress(msg, maxval) + else: + return NonInteractiveProgress(msg, maxval) + +def pluralise(singular, plural, qty): + if(qty == 1): + return singular % qty + else: + return plural % qty + + +class InteractConsoleLogFilter(logging.Filter): + def __init__(self, tf, format): + self.tf = tf + self.format = format + + def filter(self, record): + if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")): + return False + self.tf.clearFooter() + return True + +class TerminalFilter(object): + rows = 25 + columns = 80 + + def sigwinch_handle(self, signum, frame): + self.rows, self.columns = self.getTerminalColumns() + if self._sigwinch_default: + self._sigwinch_default(signum, frame) + + def getTerminalColumns(self): + def ioctl_GWINSZ(fd): + try: + cr = struct.unpack('hh', fcntl.ioctl(fd, self.termios.TIOCGWINSZ, '1234')) + except: + return None + return cr + cr = ioctl_GWINSZ(sys.stdout.fileno()) + if not cr: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = ioctl_GWINSZ(fd) + os.close(fd) + except: + pass + if not cr: + try: + cr = (env['LINES'], env['COLUMNS']) + except: + cr = (25, 80) + return cr + + def __init__(self, main, helper, console, errconsole, format, quiet): + self.main = main + self.helper = helper + self.cuu = None + self.stdinbackup = None + self.interactive = sys.stdout.isatty() + self.footer_present = False + self.lastpids = [] + self.lasttime = None + self.quiet = quiet + + if not self.interactive: + return + + try: + import curses + except ImportError: + sys.exit("FATAL: The knotty ui could not load the required curses python module.") + + import termios + self.curses = curses + self.termios = termios + try: + fd = sys.stdin.fileno() + self.stdinbackup = termios.tcgetattr(fd) + new = copy.deepcopy(self.stdinbackup) + new[3] = new[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, new) + curses.setupterm() + if curses.tigetnum("colors") > 2: + format.enable_color() + self.ed = curses.tigetstr("ed") + if self.ed: + self.cuu = curses.tigetstr("cuu") + try: + self._sigwinch_default = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, self.sigwinch_handle) + except: + pass + self.rows, self.columns = self.getTerminalColumns() + except: + self.cuu = None + if not self.cuu: + self.interactive = False + bb.note("Unable to use interactive mode for this terminal, using fallback") + return + if console: + console.addFilter(InteractConsoleLogFilter(self, format)) + if errconsole: + errconsole.addFilter(InteractConsoleLogFilter(self, format)) + + self.main_progress = None + + def clearFooter(self): + if self.footer_present: + lines = self.footer_present + sys.stdout.buffer.write(self.curses.tparm(self.cuu, lines)) + sys.stdout.buffer.write(self.curses.tparm(self.ed)) + sys.stdout.flush() + self.footer_present = False + + def updateFooter(self): + if not self.cuu: + return + activetasks = self.helper.running_tasks + failedtasks = self.helper.failed_tasks + runningpids = self.helper.running_pids + currenttime = time.time() + if not self.lasttime or (currenttime - self.lasttime > 5): + self.helper.needUpdate = True + self.lasttime = currenttime + if self.footer_present and not self.helper.needUpdate: + return + self.helper.needUpdate = False + if self.footer_present: + self.clearFooter() + if (not self.helper.tasknumber_total or self.helper.tasknumber_current == self.helper.tasknumber_total) and not len(activetasks): + return + tasks = [] + for t in runningpids: + progress = activetasks[t].get("progress", None) + if progress is not None: + pbar = activetasks[t].get("progressbar", None) + rate = activetasks[t].get("rate", None) + start_time = activetasks[t].get("starttime", None) + if not pbar or pbar.bouncing != (progress < 0): + if progress < 0: + pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.BouncingSlider(), ''], extrapos=2, resize_handler=self.sigwinch_handle) + pbar.bouncing = True + else: + pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=4, resize_handler=self.sigwinch_handle) + pbar.bouncing = False + activetasks[t]["progressbar"] = pbar + tasks.append((pbar, progress, rate, start_time)) + else: + start_time = activetasks[t].get("starttime", None) + if start_time: + tasks.append("%s - %ds (pid %s)" % (activetasks[t]["title"], currenttime - start_time, t)) + else: + tasks.append("%s (pid %s)" % (activetasks[t]["title"], t)) + + if self.main.shutdown: + content = "Waiting for %s running tasks to finish:" % len(activetasks) + print(content) + else: + if self.quiet: + content = "Running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total) + elif not len(activetasks): + content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total) + else: + content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total) + maxtask = self.helper.tasknumber_total + if not self.main_progress or self.main_progress.maxval != maxtask: + widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()] + self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets, resize_handler=self.sigwinch_handle) + self.main_progress.start(False) + self.main_progress.setmessage(content) + progress = self.helper.tasknumber_current - 1 + if progress < 0: + progress = 0 + content = self.main_progress.update(progress) + print('') + lines = 1 + int(len(content) / (self.columns + 1)) + if self.quiet == 0: + for tasknum, task in enumerate(tasks[:(self.rows - 2)]): + if isinstance(task, tuple): + pbar, progress, rate, start_time = task + if not pbar.start_time: + pbar.start(False) + if start_time: + pbar.start_time = start_time + pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1])) + if progress > -1: + pbar.setextra(rate) + content = pbar.update(progress) + else: + content = pbar.update(1) + print('') + else: + content = "%s: %s" % (tasknum, task) + print(content) + lines = lines + 1 + int(len(content) / (self.columns + 1)) + self.footer_present = lines + self.lastpids = runningpids[:] + self.lastcount = self.helper.tasknumber_current + + def finish(self): + if self.stdinbackup: + fd = sys.stdin.fileno() + self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup) + +def print_event_log(event, includelogs, loglines, termfilter): + # FIXME refactor this out further + logfile = event.logfile + if logfile and os.path.exists(logfile): + termfilter.clearFooter() + bb.error("Logfile of failure stored in: %s" % logfile) + if includelogs and not event.errprinted: + print("Log data follows:") + f = open(logfile, "r") + lines = [] + while True: + l = f.readline() + if l == '': + break + l = l.rstrip() + if loglines: + lines.append(' | %s' % l) + if len(lines) > int(loglines): + lines.pop(0) + else: + print('| %s' % l) + f.close() + if lines: + for line in lines: + print(line) + +def _log_settings_from_server(server, observe_only): + # Get values of variables which control our output + includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error) + raise BaseException(error) + loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error) + raise BaseException(error) + if observe_only: + cmd = 'getVariable' + else: + cmd = 'getSetVariable' + consolelogfile, error = server.runCommand([cmd, "BB_CONSOLELOG"]) + if error: + logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error) + raise BaseException(error) + return includelogs, loglines, consolelogfile + +_evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.LogRecord", + "bb.build.TaskFailed", "bb.build.TaskBase", "bb.event.ParseStarted", + "bb.event.ParseProgress", "bb.event.ParseCompleted", "bb.event.CacheLoadStarted", + "bb.event.CacheLoadProgress", "bb.event.CacheLoadCompleted", "bb.command.CommandFailed", + "bb.command.CommandExit", "bb.command.CommandCompleted", "bb.cooker.CookerExit", + "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted", + "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed", + "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent", + "bb.build.TaskProgress", "bb.event.ProcessStarted", "bb.event.ProcessProgress", "bb.event.ProcessFinished"] + +def main(server, eventHandler, params, tf = TerminalFilter): + + if not params.observe_only: + params.updateToServer(server, os.environ.copy()) + + includelogs, loglines, consolelogfile = _log_settings_from_server(server, params.observe_only) + + if sys.stdin.isatty() and sys.stdout.isatty(): + log_exec_tty = True + else: + log_exec_tty = False + + helper = uihelper.BBUIHelper() + + console = logging.StreamHandler(sys.stdout) + errconsole = logging.StreamHandler(sys.stderr) + format_str = "%(levelname)s: %(message)s" + format = bb.msg.BBLogFormatter(format_str) + if params.options.quiet == 0: + forcelevel = None + elif params.options.quiet > 2: + forcelevel = bb.msg.BBLogFormatter.ERROR + else: + forcelevel = bb.msg.BBLogFormatter.WARNING + bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel) + bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr) + console.setFormatter(format) + errconsole.setFormatter(format) + if not bb.msg.has_console_handler(logger): + logger.addHandler(console) + logger.addHandler(errconsole) + + bb.utils.set_process_name("KnottyUI") + + if params.options.remote_server and params.options.kill_server: + server.terminateServer() + return + + consolelog = None + if consolelogfile and not params.options.show_environment and not params.options.show_versions: + bb.utils.mkdirhier(os.path.dirname(consolelogfile)) + conlogformat = bb.msg.BBLogFormatter(format_str) + consolelog = logging.FileHandler(consolelogfile) + bb.msg.addDefaultlogFilter(consolelog) + consolelog.setFormatter(conlogformat) + logger.addHandler(consolelog) + loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log') + bb.utils.remove(loglink) + try: + os.symlink(os.path.basename(consolelogfile), loglink) + except OSError: + pass + + llevel, debug_domains = bb.msg.constructLogOptions() + server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list]) + + universe = False + if not params.observe_only: + params.updateFromServer(server) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + if cmdline['action'][0] == "buildTargets" and "universe" in cmdline['action'][1]: + universe = True + + ret, error = server.runCommand(cmdline['action']) + if error: + logger.error("Command '%s' failed: %s" % (cmdline, error)) + return 1 + elif ret != True: + logger.error("Command '%s' failed: returned %s" % (cmdline, ret)) + return 1 + + + parseprogress = None + cacheprogress = None + main.shutdown = 0 + interrupted = False + return_value = 0 + errors = 0 + warnings = 0 + taskfailures = [] + + termfilter = tf(main, helper, console, errconsole, format, params.options.quiet) + atexit.register(termfilter.finish) + + while True: + try: + event = eventHandler.waitEvent(0) + if event is None: + if main.shutdown > 1: + break + termfilter.updateFooter() + event = eventHandler.waitEvent(0.25) + if event is None: + continue + helper.eventHandler(event) + if isinstance(event, bb.runqueue.runQueueExitWait): + if not main.shutdown: + main.shutdown = 1 + continue + if isinstance(event, bb.event.LogExecTTY): + if log_exec_tty: + tries = event.retries + while tries: + print("Trying to run: %s" % event.prog) + if os.system(event.prog) == 0: + break + time.sleep(event.sleep_delay) + tries -= 1 + if tries: + continue + logger.warning(event.msg) + continue + + if isinstance(event, logging.LogRecord): + if event.levelno >= format.ERROR: + errors = errors + 1 + return_value = 1 + elif event.levelno == format.WARNING: + warnings = warnings + 1 + + if event.taskpid != 0: + # For "normal" logging conditions, don't show note logs from tasks + # but do show them if the user has changed the default log level to + # include verbose/debug messages + if event.levelno <= format.NOTE and (event.levelno < llevel or (event.levelno == format.NOTE and llevel != format.VERBOSE)): + continue + + # Prefix task messages with recipe/task + if event.taskpid in helper.running_tasks and event.levelno != format.PLAIN: + taskinfo = helper.running_tasks[event.taskpid] + event.msg = taskinfo['title'] + ': ' + event.msg + if hasattr(event, 'fn'): + event.msg = event.fn + ': ' + event.msg + logger.handle(event) + continue + + if isinstance(event, bb.build.TaskFailedSilent): + logger.warning("Logfile for failed setscene task is %s" % event.logfile) + continue + if isinstance(event, bb.build.TaskFailed): + return_value = 1 + print_event_log(event, includelogs, loglines, termfilter) + if isinstance(event, bb.build.TaskBase): + logger.info(event._message) + continue + if isinstance(event, bb.event.ParseStarted): + if params.options.quiet > 1: + continue + if event.total == 0: + continue + parseprogress = new_progress("Parsing recipes", event.total).start() + continue + if isinstance(event, bb.event.ParseProgress): + if params.options.quiet > 1: + continue + if parseprogress: + parseprogress.update(event.current) + else: + bb.warn("Got ParseProgress event for parsing that never started?") + continue + if isinstance(event, bb.event.ParseCompleted): + if params.options.quiet > 1: + continue + if not parseprogress: + continue + parseprogress.finish() + pasreprogress = None + if params.options.quiet == 0: + print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." + % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors))) + continue + + if isinstance(event, bb.event.CacheLoadStarted): + if params.options.quiet > 1: + continue + cacheprogress = new_progress("Loading cache", event.total).start() + continue + if isinstance(event, bb.event.CacheLoadProgress): + if params.options.quiet > 1: + continue + cacheprogress.update(event.current) + continue + if isinstance(event, bb.event.CacheLoadCompleted): + if params.options.quiet > 1: + continue + cacheprogress.finish() + if params.options.quiet == 0: + print("Loaded %d entries from dependency cache." % event.num_entries) + continue + + if isinstance(event, bb.command.CommandFailed): + return_value = event.exitcode + if event.error: + errors = errors + 1 + logger.error(str(event)) + main.shutdown = 2 + continue + if isinstance(event, bb.command.CommandExit): + if not return_value: + return_value = event.exitcode + continue + if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)): + main.shutdown = 2 + continue + if isinstance(event, bb.event.MultipleProviders): + logger.info(str(event)) + continue + if isinstance(event, bb.event.NoProvider): + # For universe builds, only show these as warnings, not errors + if not universe: + return_value = 1 + errors = errors + 1 + logger.error(str(event)) + else: + logger.warning(str(event)) + continue + + if isinstance(event, bb.runqueue.sceneQueueTaskStarted): + logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring)) + continue + + if isinstance(event, bb.runqueue.runQueueTaskStarted): + if event.noexec: + tasktype = 'noexec task' + else: + tasktype = 'task' + logger.info("Running %s %d of %d (%s)", + tasktype, + event.stats.completed + event.stats.active + + event.stats.failed + 1, + event.stats.total, event.taskstring) + continue + + if isinstance(event, bb.runqueue.runQueueTaskFailed): + return_value = 1 + taskfailures.append(event.taskstring) + logger.error(str(event)) + continue + + if isinstance(event, bb.runqueue.sceneQueueTaskFailed): + logger.warning(str(event)) + continue + + if isinstance(event, bb.event.DepTreeGenerated): + continue + + if isinstance(event, bb.event.ProcessStarted): + if params.options.quiet > 1: + continue + parseprogress = new_progress(event.processname, event.total) + parseprogress.start(False) + continue + if isinstance(event, bb.event.ProcessProgress): + if params.options.quiet > 1: + continue + if parseprogress: + parseprogress.update(event.progress) + else: + bb.warn("Got ProcessProgress event for someting that never started?") + continue + if isinstance(event, bb.event.ProcessFinished): + if params.options.quiet > 1: + continue + if parseprogress: + parseprogress.finish() + parseprogress = None + continue + + # ignore + if isinstance(event, (bb.event.BuildBase, + bb.event.MetadataEvent, + bb.event.StampUpdate, + bb.event.ConfigParsed, + bb.event.MultiConfigParsed, + bb.event.RecipeParsed, + bb.event.RecipePreFinalise, + bb.runqueue.runQueueEvent, + bb.event.OperationStarted, + bb.event.OperationCompleted, + bb.event.OperationProgress, + bb.event.DiskFull, + bb.event.HeartbeatEvent, + bb.build.TaskProgress)): + continue + + logger.error("Unknown event: %s", event) + + except EnvironmentError as ioerror: + termfilter.clearFooter() + # ignore interrupted io + if ioerror.args[0] == 4: + continue + sys.stderr.write(str(ioerror)) + if not params.observe_only: + _, error = server.runCommand(["stateForceShutdown"]) + main.shutdown = 2 + except KeyboardInterrupt: + termfilter.clearFooter() + if params.observe_only: + print("\nKeyboard Interrupt, exiting observer...") + main.shutdown = 2 + if not params.observe_only and main.shutdown == 1: + print("\nSecond Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + logger.error("Unable to cleanly stop: %s" % error) + if not params.observe_only and main.shutdown == 0: + print("\nKeyboard Interrupt, closing down...\n") + interrupted = True + _, error = server.runCommand(["stateShutdown"]) + if error: + logger.error("Unable to cleanly shutdown: %s" % error) + main.shutdown = main.shutdown + 1 + pass + except Exception as e: + import traceback + sys.stderr.write(traceback.format_exc()) + if not params.observe_only: + _, error = server.runCommand(["stateForceShutdown"]) + main.shutdown = 2 + return_value = 1 + try: + termfilter.clearFooter() + summary = "" + if taskfailures: + summary += pluralise("\nSummary: %s task failed:", + "\nSummary: %s tasks failed:", len(taskfailures)) + for failure in taskfailures: + summary += "\n %s" % failure + if warnings: + summary += pluralise("\nSummary: There was %s WARNING message shown.", + "\nSummary: There were %s WARNING messages shown.", warnings) + if return_value and errors: + summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.", + "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors) + if summary and params.options.quiet == 0: + print(summary) + + if interrupted: + print("Execution was interrupted, returning a non-zero exit code.") + if return_value == 0: + return_value = 1 + except IOError as e: + import errno + if e.errno == errno.EPIPE: + pass + + if consolelog: + logger.removeHandler(consolelog) + consolelog.close() + + return return_value diff --git a/poky/bitbake/lib/bb/ui/ncurses.py b/poky/bitbake/lib/bb/ui/ncurses.py new file mode 100644 index 000000000..8690c529c --- /dev/null +++ b/poky/bitbake/lib/bb/ui/ncurses.py @@ -0,0 +1,373 @@ +# +# BitBake Curses UI Implementation +# +# Implements an ncurses frontend for the BitBake utility. +# +# Copyright (C) 2006 Michael 'Mickey' Lauer +# Copyright (C) 2006-2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + We have the following windows: + + 1.) Main Window: Shows what we are ultimately building and how far we are. Includes status bar + 2.) Thread Activity Window: Shows one status line for every concurrent bitbake thread. + 3.) Command Line Window: Contains an interactive command line where you can interact w/ Bitbake. + + Basic window layout is like that: + + |---------------------------------------------------------| + | <Main Window> | <Thread Activity Window> | + | | 0: foo do_compile complete| + | Building Gtk+-2.6.10 | 1: bar do_patch complete | + | Status: 60% | ... | + | | ... | + | | ... | + |---------------------------------------------------------| + |<Command Line Window> | + |>>> which virtual/kernel | + |openzaurus-kernel | + |>>> _ | + |---------------------------------------------------------| + +""" + + + +import logging +import os, sys, itertools, time, subprocess + +try: + import curses +except ImportError: + sys.exit("FATAL: The ncurses ui could not load the required curses python module.") + +import bb +import xmlrpc.client +from bb import ui +from bb.ui import uihelper + +parsespin = itertools.cycle( r'|/-\\' ) + +X = 0 +Y = 1 +WIDTH = 2 +HEIGHT = 3 + +MAXSTATUSLENGTH = 32 + +class NCursesUI: + """ + NCurses UI Class + """ + class Window: + """Base Window Class""" + def __init__( self, x, y, width, height, fg=curses.COLOR_BLACK, bg=curses.COLOR_WHITE ): + self.win = curses.newwin( height, width, y, x ) + self.dimensions = ( x, y, width, height ) + """ + if curses.has_colors(): + color = 1 + curses.init_pair( color, fg, bg ) + self.win.bkgdset( ord(' '), curses.color_pair(color) ) + else: + self.win.bkgdset( ord(' '), curses.A_BOLD ) + """ + self.erase() + self.setScrolling() + self.win.noutrefresh() + + def erase( self ): + self.win.erase() + + def setScrolling( self, b = True ): + self.win.scrollok( b ) + self.win.idlok( b ) + + def setBoxed( self ): + self.boxed = True + self.win.box() + self.win.noutrefresh() + + def setText( self, x, y, text, *args ): + self.win.addstr( y, x, text, *args ) + self.win.noutrefresh() + + def appendText( self, text, *args ): + self.win.addstr( text, *args ) + self.win.noutrefresh() + + def drawHline( self, y ): + self.win.hline( y, 0, curses.ACS_HLINE, self.dimensions[WIDTH] ) + self.win.noutrefresh() + + class DecoratedWindow( Window ): + """Base class for windows with a box and a title bar""" + def __init__( self, title, x, y, width, height, fg=curses.COLOR_BLACK, bg=curses.COLOR_WHITE ): + NCursesUI.Window.__init__( self, x+1, y+3, width-2, height-4, fg, bg ) + self.decoration = NCursesUI.Window( x, y, width, height, fg, bg ) + self.decoration.setBoxed() + self.decoration.win.hline( 2, 1, curses.ACS_HLINE, width-2 ) + self.setTitle( title ) + + def setTitle( self, title ): + self.decoration.setText( 1, 1, title.center( self.dimensions[WIDTH]-2 ), curses.A_BOLD ) + + #-------------------------------------------------------------------------# +# class TitleWindow( Window ): + #-------------------------------------------------------------------------# +# """Title Window""" +# def __init__( self, x, y, width, height ): +# NCursesUI.Window.__init__( self, x, y, width, height ) +# version = bb.__version__ +# title = "BitBake %s" % version +# credit = "(C) 2003-2007 Team BitBake" +# #self.win.hline( 2, 1, curses.ACS_HLINE, width-2 ) +# self.win.border() +# self.setText( 1, 1, title.center( self.dimensions[WIDTH]-2 ), curses.A_BOLD ) +# self.setText( 1, 2, credit.center( self.dimensions[WIDTH]-2 ), curses.A_BOLD ) + + #-------------------------------------------------------------------------# + class ThreadActivityWindow( DecoratedWindow ): + #-------------------------------------------------------------------------# + """Thread Activity Window""" + def __init__( self, x, y, width, height ): + NCursesUI.DecoratedWindow.__init__( self, "Thread Activity", x, y, width, height ) + + def setStatus( self, thread, text ): + line = "%02d: %s" % ( thread, text ) + width = self.dimensions[WIDTH] + if ( len(line) > width ): + line = line[:width-3] + "..." + else: + line = line.ljust( width ) + self.setText( 0, thread, line ) + + #-------------------------------------------------------------------------# + class MainWindow( DecoratedWindow ): + #-------------------------------------------------------------------------# + """Main Window""" + def __init__( self, x, y, width, height ): + self.StatusPosition = width - MAXSTATUSLENGTH + NCursesUI.DecoratedWindow.__init__( self, None, x, y, width, height ) + curses.nl() + + def setTitle( self, title ): + title = "BitBake %s" % bb.__version__ + self.decoration.setText( 2, 1, title, curses.A_BOLD ) + self.decoration.setText( self.StatusPosition - 8, 1, "Status:", curses.A_BOLD ) + + def setStatus(self, status): + while len(status) < MAXSTATUSLENGTH: + status = status + " " + self.decoration.setText( self.StatusPosition, 1, status, curses.A_BOLD ) + + + #-------------------------------------------------------------------------# + class ShellOutputWindow( DecoratedWindow ): + #-------------------------------------------------------------------------# + """Interactive Command Line Output""" + def __init__( self, x, y, width, height ): + NCursesUI.DecoratedWindow.__init__( self, "Command Line Window", x, y, width, height ) + + #-------------------------------------------------------------------------# + class ShellInputWindow( Window ): + #-------------------------------------------------------------------------# + """Interactive Command Line Input""" + def __init__( self, x, y, width, height ): + NCursesUI.Window.__init__( self, x, y, width, height ) + +# put that to the top again from curses.textpad import Textbox +# self.textbox = Textbox( self.win ) +# t = threading.Thread() +# t.run = self.textbox.edit +# t.start() + + #-------------------------------------------------------------------------# + def main(self, stdscr, server, eventHandler, params): + #-------------------------------------------------------------------------# + height, width = stdscr.getmaxyx() + + # for now split it like that: + # MAIN_y + THREAD_y = 2/3 screen at the top + # MAIN_x = 2/3 left, THREAD_y = 1/3 right + # CLI_y = 1/3 of screen at the bottom + # CLI_x = full + + main_left = 0 + main_top = 0 + main_height = ( height // 3 * 2 ) + main_width = ( width // 3 ) * 2 + clo_left = main_left + clo_top = main_top + main_height + clo_height = height - main_height - main_top - 1 + clo_width = width + cli_left = main_left + cli_top = clo_top + clo_height + cli_height = 1 + cli_width = width + thread_left = main_left + main_width + thread_top = main_top + thread_height = main_height + thread_width = width - main_width + + #tw = self.TitleWindow( 0, 0, width, main_top ) + mw = self.MainWindow( main_left, main_top, main_width, main_height ) + taw = self.ThreadActivityWindow( thread_left, thread_top, thread_width, thread_height ) + clo = self.ShellOutputWindow( clo_left, clo_top, clo_width, clo_height ) + cli = self.ShellInputWindow( cli_left, cli_top, cli_width, cli_height ) + cli.setText( 0, 0, "BB>" ) + + mw.setStatus("Idle") + + helper = uihelper.BBUIHelper() + shutdown = 0 + + try: + params.updateFromServer(server) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + cmdline = cmdline['action'] + ret, error = server.runCommand(cmdline) + if error: + print("Error running command '%s': %s" % (cmdline, error)) + return + elif ret != True: + print("Couldn't get default commandlind! %s" % ret) + return + except xmlrpc.client.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) + return + + exitflag = False + while not exitflag: + try: + event = eventHandler.waitEvent(0.25) + if not event: + continue + + helper.eventHandler(event) + if isinstance(event, bb.build.TaskBase): + mw.appendText("NOTE: %s\n" % event._message) + if isinstance(event, logging.LogRecord): + mw.appendText(logging.getLevelName(event.levelno) + ': ' + event.getMessage() + '\n') + + if isinstance(event, bb.event.CacheLoadStarted): + self.parse_total = event.total + if isinstance(event, bb.event.CacheLoadProgress): + x = event.current + y = self.parse_total + mw.setStatus("Loading Cache: %s [%2d %%]" % ( next(parsespin), x*100/y ) ) + if isinstance(event, bb.event.CacheLoadCompleted): + mw.setStatus("Idle") + mw.appendText("Loaded %d entries from dependency cache.\n" + % ( event.num_entries)) + + if isinstance(event, bb.event.ParseStarted): + self.parse_total = event.total + if isinstance(event, bb.event.ParseProgress): + x = event.current + y = self.parse_total + mw.setStatus("Parsing Recipes: %s [%2d %%]" % ( next(parsespin), x*100/y ) ) + if isinstance(event, bb.event.ParseCompleted): + mw.setStatus("Idle") + mw.appendText("Parsing finished. %d cached, %d parsed, %d skipped, %d masked.\n" + % ( event.cached, event.parsed, event.skipped, event.masked )) + +# if isinstance(event, bb.build.TaskFailed): +# if event.logfile: +# if data.getVar("BBINCLUDELOGS", d): +# bb.error("log data follows (%s)" % logfile) +# number_of_lines = data.getVar("BBINCLUDELOGS_LINES", d) +# if number_of_lines: +# subprocess.check_call('tail -n%s %s' % (number_of_lines, logfile), shell=True) +# else: +# f = open(logfile, "r") +# while True: +# l = f.readline() +# if l == '': +# break +# l = l.rstrip() +# print '| %s' % l +# f.close() +# else: +# bb.error("see log in %s" % logfile) + + if isinstance(event, bb.command.CommandCompleted): + # stop so the user can see the result of the build, but + # also allow them to now exit with a single ^C + shutdown = 2 + if isinstance(event, bb.command.CommandFailed): + mw.appendText(str(event)) + time.sleep(2) + exitflag = True + if isinstance(event, bb.command.CommandExit): + exitflag = True + if isinstance(event, bb.cooker.CookerExit): + exitflag = True + + if isinstance(event, bb.event.LogExecTTY): + mw.appendText('WARN: ' + event.msg + '\n') + if helper.needUpdate: + activetasks, failedtasks = helper.getTasks() + taw.erase() + taw.setText(0, 0, "") + if activetasks: + taw.appendText("Active Tasks:\n") + for task in activetasks.values(): + taw.appendText(task["title"] + '\n') + if failedtasks: + taw.appendText("Failed Tasks:\n") + for task in failedtasks: + taw.appendText(task["title"] + '\n') + + curses.doupdate() + except EnvironmentError as ioerror: + # ignore interrupted io + if ioerror.args[0] == 4: + pass + + except KeyboardInterrupt: + if shutdown == 2: + mw.appendText("Third Keyboard Interrupt, exit.\n") + exitflag = True + if shutdown == 1: + mw.appendText("Second Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + print("Unable to cleanly stop: %s" % error) + if shutdown == 0: + mw.appendText("Keyboard Interrupt, closing down...\n") + _, error = server.runCommand(["stateShutdown"]) + if error: + print("Unable to cleanly shutdown: %s" % error) + shutdown = shutdown + 1 + pass + +def main(server, eventHandler, params): + if not os.isatty(sys.stdout.fileno()): + print("FATAL: Unable to run 'ncurses' UI without a TTY.") + return + ui = NCursesUI() + try: + curses.wrapper(ui.main, server, eventHandler, params) + except: + import traceback + traceback.print_exc() diff --git a/poky/bitbake/lib/bb/ui/taskexp.py b/poky/bitbake/lib/bb/ui/taskexp.py new file mode 100644 index 000000000..0e8e9d4cf --- /dev/null +++ b/poky/bitbake/lib/bb/ui/taskexp.py @@ -0,0 +1,328 @@ +# +# BitBake Graphical GTK based Dependency Explorer +# +# Copyright (C) 2007 Ross Burton +# Copyright (C) 2007 - 2008 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk, Gdk, GObject +from multiprocessing import Queue +import threading +from xmlrpc import client +import time +import bb +import bb.event + +# Package Model +(COL_PKG_NAME) = (0) + +# Dependency Model +(TYPE_DEP, TYPE_RDEP) = (0, 1) +(COL_DEP_TYPE, COL_DEP_PARENT, COL_DEP_PACKAGE) = (0, 1, 2) + + +class PackageDepView(Gtk.TreeView): + def __init__(self, model, dep_type, label): + Gtk.TreeView.__init__(self) + self.current = None + self.dep_type = dep_type + self.filter_model = model.filter_new() + self.filter_model.set_visible_func(self._filter, data=None) + self.set_model(self.filter_model) + self.append_column(Gtk.TreeViewColumn(label, Gtk.CellRendererText(), text=COL_DEP_PACKAGE)) + + def _filter(self, model, iter, data): + this_type = model[iter][COL_DEP_TYPE] + package = model[iter][COL_DEP_PARENT] + if this_type != self.dep_type: return False + return package == self.current + + def set_current_package(self, package): + self.current = package + self.filter_model.refilter() + + +class PackageReverseDepView(Gtk.TreeView): + def __init__(self, model, label): + Gtk.TreeView.__init__(self) + self.current = None + self.filter_model = model.filter_new() + self.filter_model.set_visible_func(self._filter) + self.sort_model = self.filter_model.sort_new_with_model() + self.sort_model.set_sort_column_id(COL_DEP_PARENT, Gtk.SortType.ASCENDING) + self.set_model(self.sort_model) + self.append_column(Gtk.TreeViewColumn(label, Gtk.CellRendererText(), text=COL_DEP_PARENT)) + + def _filter(self, model, iter, data): + package = model[iter][COL_DEP_PACKAGE] + return package == self.current + + def set_current_package(self, package): + self.current = package + self.filter_model.refilter() + + +class DepExplorer(Gtk.Window): + def __init__(self): + Gtk.Window.__init__(self) + self.set_title("Task Dependency Explorer") + self.set_default_size(500, 500) + self.connect("delete-event", Gtk.main_quit) + + # Create the data models + self.pkg_model = Gtk.ListStore(GObject.TYPE_STRING) + self.pkg_model.set_sort_column_id(COL_PKG_NAME, Gtk.SortType.ASCENDING) + self.depends_model = Gtk.ListStore(GObject.TYPE_INT, GObject.TYPE_STRING, GObject.TYPE_STRING) + self.depends_model.set_sort_column_id(COL_DEP_PACKAGE, Gtk.SortType.ASCENDING) + + pane = Gtk.HPaned() + pane.set_position(250) + self.add(pane) + + # The master list of packages + scrolled = Gtk.ScrolledWindow() + scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC) + scrolled.set_shadow_type(Gtk.ShadowType.IN) + + self.pkg_treeview = Gtk.TreeView(self.pkg_model) + self.pkg_treeview.get_selection().connect("changed", self.on_cursor_changed) + column = Gtk.TreeViewColumn("Package", Gtk.CellRendererText(), text=COL_PKG_NAME) + self.pkg_treeview.append_column(column) + pane.add1(scrolled) + scrolled.add(self.pkg_treeview) + + box = Gtk.VBox(homogeneous=True, spacing=4) + + # Task Depends + scrolled = Gtk.ScrolledWindow() + scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC) + scrolled.set_shadow_type(Gtk.ShadowType.IN) + self.dep_treeview = PackageDepView(self.depends_model, TYPE_DEP, "Dependencies") + self.dep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE) + scrolled.add(self.dep_treeview) + box.add(scrolled) + pane.add2(box) + + # Reverse Task Depends + scrolled = Gtk.ScrolledWindow() + scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC) + scrolled.set_shadow_type(Gtk.ShadowType.IN) + self.revdep_treeview = PackageReverseDepView(self.depends_model, "Dependent Tasks") + self.revdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PARENT) + scrolled.add(self.revdep_treeview) + box.add(scrolled) + pane.add2(box) + + self.show_all() + + def on_package_activated(self, treeview, path, column, data_col): + model = treeview.get_model() + package = model.get_value(model.get_iter(path), data_col) + + pkg_path = [] + def finder(model, path, iter, needle): + package = model.get_value(iter, COL_PKG_NAME) + if package == needle: + pkg_path.append(path) + return True + else: + return False + self.pkg_model.foreach(finder, package) + if pkg_path: + self.pkg_treeview.get_selection().select_path(pkg_path[0]) + self.pkg_treeview.scroll_to_cell(pkg_path[0]) + + def on_cursor_changed(self, selection): + (model, it) = selection.get_selected() + if it is None: + current_package = None + else: + current_package = model.get_value(it, COL_PKG_NAME) + self.dep_treeview.set_current_package(current_package) + self.revdep_treeview.set_current_package(current_package) + + + def parse(self, depgraph): + for task in depgraph["tdepends"]: + self.pkg_model.insert(0, (task,)) + for depend in depgraph["tdepends"][task]: + self.depends_model.insert (0, (TYPE_DEP, task, depend)) + + +class gtkthread(threading.Thread): + quit = threading.Event() + def __init__(self, shutdown): + threading.Thread.__init__(self) + self.setDaemon(True) + self.shutdown = shutdown + if not Gtk.init_check()[0]: + sys.stderr.write("Gtk+ init failed. Make sure DISPLAY variable is set.\n") + gtkthread.quit.set() + + def run(self): + GObject.threads_init() + Gdk.threads_init() + Gtk.main() + gtkthread.quit.set() + + +def main(server, eventHandler, params): + shutdown = 0 + + gtkgui = gtkthread(shutdown) + gtkgui.start() + + try: + params.updateFromServer(server) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + print(cmdline['msg']) + return 1 + cmdline = cmdline['action'] + if not cmdline or cmdline[0] != "generateDotGraph": + print("This UI requires the -g option") + return 1 + ret, error = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]]) + if error: + print("Error running command '%s': %s" % (cmdline, error)) + return 1 + elif ret != True: + print("Error running command '%s': returned %s" % (cmdline, ret)) + return 1 + except client.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) + return + + if gtkthread.quit.isSet(): + return + + Gdk.threads_enter() + dep = DepExplorer() + bardialog = Gtk.Dialog(parent=dep, + flags=Gtk.DialogFlags.MODAL|Gtk.DialogFlags.DESTROY_WITH_PARENT) + bardialog.set_default_size(400, 50) + box = bardialog.get_content_area() + pbar = Gtk.ProgressBar() + box.pack_start(pbar, True, True, 0) + bardialog.show_all() + bardialog.connect("delete-event", Gtk.main_quit) + Gdk.threads_leave() + + progress_total = 0 + while True: + try: + event = eventHandler.waitEvent(0.25) + if gtkthread.quit.isSet(): + _, error = server.runCommand(["stateForceShutdown"]) + if error: + print('Unable to cleanly stop: %s' % error) + break + + if event is None: + continue + + if isinstance(event, bb.event.CacheLoadStarted): + progress_total = event.total + Gdk.threads_enter() + bardialog.set_title("Loading Cache") + pbar.set_fraction(0.0) + Gdk.threads_leave() + + if isinstance(event, bb.event.CacheLoadProgress): + x = event.current + Gdk.threads_enter() + pbar.set_fraction(x * 1.0 / progress_total) + Gdk.threads_leave() + continue + + if isinstance(event, bb.event.CacheLoadCompleted): + continue + + if isinstance(event, bb.event.ParseStarted): + progress_total = event.total + if progress_total == 0: + continue + Gdk.threads_enter() + pbar.set_fraction(0.0) + bardialog.set_title("Processing recipes") + Gdk.threads_leave() + + if isinstance(event, bb.event.ParseProgress): + x = event.current + Gdk.threads_enter() + pbar.set_fraction(x * 1.0 / progress_total) + Gdk.threads_leave() + continue + + if isinstance(event, bb.event.ParseCompleted): + Gdk.threads_enter() + bardialog.set_title("Generating dependency tree") + Gdk.threads_leave() + continue + + if isinstance(event, bb.event.DepTreeGenerated): + Gdk.threads_enter() + bardialog.hide() + dep.parse(event._depgraph) + Gdk.threads_leave() + + if isinstance(event, bb.command.CommandCompleted): + continue + + if isinstance(event, bb.event.NoProvider): + print(str(event)) + + _, error = server.runCommand(["stateShutdown"]) + if error: + print('Unable to cleanly shutdown: %s' % error) + break + + if isinstance(event, bb.command.CommandFailed): + print(str(event)) + return event.exitcode + + if isinstance(event, bb.command.CommandExit): + return event.exitcode + + if isinstance(event, bb.cooker.CookerExit): + break + + continue + except EnvironmentError as ioerror: + # ignore interrupted io + if ioerror.args[0] == 4: + pass + except KeyboardInterrupt: + if shutdown == 2: + print("\nThird Keyboard Interrupt, exit.\n") + break + if shutdown == 1: + print("\nSecond Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + print('Unable to cleanly stop: %s' % error) + if shutdown == 0: + print("\nKeyboard Interrupt, closing down...\n") + _, error = server.runCommand(["stateShutdown"]) + if error: + print('Unable to cleanly shutdown: %s' % error) + shutdown = shutdown + 1 + pass diff --git a/poky/bitbake/lib/bb/ui/toasterui.py b/poky/bitbake/lib/bb/ui/toasterui.py new file mode 100644 index 000000000..88cec3759 --- /dev/null +++ b/poky/bitbake/lib/bb/ui/toasterui.py @@ -0,0 +1,487 @@ +# +# BitBake ToasterUI Implementation +# based on (No)TTY UI Implementation by Richard Purdie +# +# Handling output to TTYs or files (no TTY) +# +# Copyright (C) 2006-2012 Richard Purdie +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import division +import time +import sys +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +from bb.ui import uihelper +from bb.ui.buildinfohelper import BuildInfoHelper + +import bb.msg +import logging +import os + +# pylint: disable=invalid-name +# module properties for UI modules are read by bitbake and the contract should not be broken + + +featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING, bb.cooker.CookerFeatures.SEND_SANITYEVENTS] + +logger = logging.getLogger("ToasterLogger") +interactive = sys.stdout.isatty() + +def _log_settings_from_server(server): + # Get values of variables which control our output + includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS variable: %s", error) + raise BaseException(error) + loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"]) + if error: + logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s", error) + raise BaseException(error) + consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"]) + if error: + logger.error("Unable to get the value of BB_CONSOLELOG variable: %s", error) + raise BaseException(error) + return consolelogfile + +# create a log file for a single build and direct the logger at it; +# log file name is timestamped to the millisecond (depending +# on system clock accuracy) to ensure it doesn't overlap with +# other log file names +# +# returns (log file, path to log file) for a build +def _open_build_log(log_dir): + format_str = "%(levelname)s: %(message)s" + + now = time.time() + now_ms = int((now - int(now)) * 1000) + time_str = time.strftime('build_%Y%m%d_%H%M%S', time.localtime(now)) + log_file_name = time_str + ('.%d.log' % now_ms) + build_log_file_path = os.path.join(log_dir, log_file_name) + + build_log = logging.FileHandler(build_log_file_path) + + logformat = bb.msg.BBLogFormatter(format_str) + build_log.setFormatter(logformat) + + bb.msg.addDefaultlogFilter(build_log) + logger.addHandler(build_log) + + return (build_log, build_log_file_path) + +# stop logging to the build log if it exists +def _close_build_log(build_log): + if build_log: + build_log.flush() + build_log.close() + logger.removeHandler(build_log) + +_evt_list = [ + "bb.build.TaskBase", + "bb.build.TaskFailed", + "bb.build.TaskFailedSilent", + "bb.build.TaskStarted", + "bb.build.TaskSucceeded", + "bb.command.CommandCompleted", + "bb.command.CommandExit", + "bb.command.CommandFailed", + "bb.cooker.CookerExit", + "bb.event.BuildInit", + "bb.event.BuildCompleted", + "bb.event.BuildStarted", + "bb.event.CacheLoadCompleted", + "bb.event.CacheLoadProgress", + "bb.event.CacheLoadStarted", + "bb.event.ConfigParsed", + "bb.event.DepTreeGenerated", + "bb.event.LogExecTTY", + "bb.event.MetadataEvent", + "bb.event.MultipleProviders", + "bb.event.NoProvider", + "bb.event.ParseCompleted", + "bb.event.ParseProgress", + "bb.event.ParseStarted", + "bb.event.RecipeParsed", + "bb.event.SanityCheck", + "bb.event.SanityCheckPassed", + "bb.event.TreeDataPreparationCompleted", + "bb.event.TreeDataPreparationStarted", + "bb.runqueue.runQueueTaskCompleted", + "bb.runqueue.runQueueTaskFailed", + "bb.runqueue.runQueueTaskSkipped", + "bb.runqueue.runQueueTaskStarted", + "bb.runqueue.sceneQueueTaskCompleted", + "bb.runqueue.sceneQueueTaskFailed", + "bb.runqueue.sceneQueueTaskStarted", + "logging.LogRecord"] + +def main(server, eventHandler, params): + # set to a logging.FileHandler instance when a build starts; + # see _open_build_log() + build_log = None + + # set to the log path when a build starts + build_log_file_path = None + + helper = uihelper.BBUIHelper() + + # TODO don't use log output to determine when bitbake has started + # + # WARNING: this log handler cannot be removed, as localhostbecontroller + # relies on output in the toaster_ui.log file to determine whether + # the bitbake server has started, which only happens if + # this logger is setup here (see the TODO in the loop below) + console = logging.StreamHandler(sys.stdout) + format_str = "%(levelname)s: %(message)s" + formatter = bb.msg.BBLogFormatter(format_str) + bb.msg.addDefaultlogFilter(console) + console.setFormatter(formatter) + logger.addHandler(console) + logger.setLevel(logging.INFO) + llevel, debug_domains = bb.msg.constructLogOptions() + result, error = server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list]) + if not result or error: + logger.error("can't set event mask: %s", error) + return 1 + + # verify and warn + build_history_enabled = True + inheritlist, _ = server.runCommand(["getVariable", "INHERIT"]) + + if not "buildhistory" in inheritlist.split(" "): + logger.warning("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.") + build_history_enabled = False + + if not "buildstats" in inheritlist.split(" "): + logger.warning("buildstats is not enabled. Please enable INHERIT += \"buildstats\" to generate build statistics.") + + if not params.observe_only: + params.updateFromServer(server) + params.updateToServer(server, os.environ.copy()) + cmdline = params.parseActions() + if not cmdline: + print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + return 1 + if 'msg' in cmdline and cmdline['msg']: + logger.error(cmdline['msg']) + return 1 + + ret, error = server.runCommand(cmdline['action']) + if error: + logger.error("Command '%s' failed: %s" % (cmdline, error)) + return 1 + elif ret != True: + logger.error("Command '%s' failed: returned %s" % (cmdline, ret)) + return 1 + + # set to 1 when toasterui needs to shut down + main.shutdown = 0 + + interrupted = False + return_value = 0 + errors = 0 + warnings = 0 + taskfailures = [] + first = True + + buildinfohelper = BuildInfoHelper(server, build_history_enabled, + os.getenv('TOASTER_BRBE')) + + # write our own log files into bitbake's log directory; + # we're only interested in the path to the parent directory of + # this file, as we're writing our own logs into the same directory + consolelogfile = _log_settings_from_server(server) + log_dir = os.path.dirname(consolelogfile) + bb.utils.mkdirhier(log_dir) + + while True: + try: + event = eventHandler.waitEvent(0.25) + if first: + first = False + + # TODO don't use log output to determine when bitbake has started + # + # this is the line localhostbecontroller needs to + # see in toaster_ui.log which it uses to decide whether + # the bitbake server has started... + logger.info("ToasterUI waiting for events") + + if event is None: + if main.shutdown > 0: + # if shutting down, close any open build log first + _close_build_log(build_log) + + break + continue + + helper.eventHandler(event) + + # pylint: disable=protected-access + # the code will look into the protected variables of the event; no easy way around this + + if isinstance(event, bb.event.HeartbeatEvent): + continue + + if isinstance(event, bb.event.ParseStarted): + if not (build_log and build_log_file_path): + build_log, build_log_file_path = _open_build_log(log_dir) + + buildinfohelper.store_started_build() + buildinfohelper.save_build_log_file_path(build_log_file_path) + buildinfohelper.set_recipes_to_parse(event.total) + continue + + # create a build object in buildinfohelper from either BuildInit + # (if available) or BuildStarted (for jethro and previous versions) + if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)): + if not (build_log and build_log_file_path): + build_log, build_log_file_path = _open_build_log(log_dir) + + buildinfohelper.save_build_targets(event) + buildinfohelper.save_build_log_file_path(build_log_file_path) + + # get additional data from BuildStarted + if isinstance(event, bb.event.BuildStarted): + buildinfohelper.save_build_layers_and_variables() + continue + + if isinstance(event, bb.event.ParseProgress): + buildinfohelper.set_recipes_parsed(event.current) + continue + + if isinstance(event, bb.event.ParseCompleted): + buildinfohelper.set_recipes_parsed(event.total) + continue + + if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): + buildinfohelper.update_and_store_task(event) + logger.info("Logfile for task %s", event.logfile) + continue + + if isinstance(event, bb.build.TaskBase): + logger.info(event._message) + + if isinstance(event, bb.event.LogExecTTY): + logger.info(event.msg) + continue + + if isinstance(event, logging.LogRecord): + if event.levelno == -1: + event.levelno = formatter.ERROR + + buildinfohelper.store_log_event(event) + + if event.levelno >= formatter.ERROR: + errors = errors + 1 + elif event.levelno == formatter.WARNING: + warnings = warnings + 1 + + # For "normal" logging conditions, don't show note logs from tasks + # but do show them if the user has changed the default log level to + # include verbose/debug messages + if event.taskpid != 0 and event.levelno <= formatter.NOTE: + continue + + logger.handle(event) + continue + + if isinstance(event, bb.build.TaskFailed): + buildinfohelper.update_and_store_task(event) + logfile = event.logfile + if logfile and os.path.exists(logfile): + bb.error("Logfile of failure stored in: %s" % logfile) + continue + + # these events are unprocessed now, but may be used in the future to log + # timing and error informations from the parsing phase in Toaster + if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): + continue + if isinstance(event, bb.event.CacheLoadStarted): + continue + if isinstance(event, bb.event.CacheLoadProgress): + continue + if isinstance(event, bb.event.CacheLoadCompleted): + continue + if isinstance(event, bb.event.MultipleProviders): + logger.info(str(event)) + continue + + if isinstance(event, bb.event.NoProvider): + errors = errors + 1 + text = str(event) + logger.error(text) + buildinfohelper.store_log_error(text) + continue + + if isinstance(event, bb.event.ConfigParsed): + continue + if isinstance(event, bb.event.RecipeParsed): + continue + + # end of saved events + + if isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)): + buildinfohelper.store_started_task(event) + continue + + if isinstance(event, bb.runqueue.runQueueTaskCompleted): + buildinfohelper.update_and_store_task(event) + continue + + if isinstance(event, bb.runqueue.runQueueTaskFailed): + buildinfohelper.update_and_store_task(event) + taskfailures.append(event.taskstring) + logger.error(str(event)) + continue + + if isinstance(event, (bb.runqueue.sceneQueueTaskCompleted, bb.runqueue.sceneQueueTaskFailed)): + buildinfohelper.update_and_store_task(event) + continue + + + if isinstance(event, (bb.event.TreeDataPreparationStarted, bb.event.TreeDataPreparationCompleted)): + continue + + if isinstance(event, (bb.event.BuildCompleted, bb.command.CommandFailed)): + + errorcode = 0 + if isinstance(event, bb.command.CommandFailed): + errors += 1 + errorcode = 1 + logger.error(str(event)) + elif isinstance(event, bb.event.BuildCompleted): + buildinfohelper.scan_image_artifacts() + buildinfohelper.clone_required_sdk_artifacts() + + # turn off logging to the current build log + _close_build_log(build_log) + + # reset ready for next BuildStarted + build_log = None + + # update the build info helper on BuildCompleted, not on CommandXXX + buildinfohelper.update_build_information(event, errors, warnings, taskfailures) + + brbe = buildinfohelper.brbe + buildinfohelper.close(errorcode) + + # we start a new build info + if params.observe_only: + logger.debug("ToasterUI prepared for new build") + errors = 0 + warnings = 0 + taskfailures = [] + buildinfohelper = BuildInfoHelper(server, build_history_enabled) + else: + main.shutdown = 1 + + logger.info("ToasterUI build done, brbe: %s", brbe) + continue + + if isinstance(event, (bb.command.CommandCompleted, + bb.command.CommandFailed, + bb.command.CommandExit)): + if params.observe_only: + errorcode = 0 + else: + main.shutdown = 1 + + continue + + if isinstance(event, bb.event.MetadataEvent): + if event.type == "SinglePackageInfo": + buildinfohelper.store_build_package_information(event) + elif event.type == "LayerInfo": + buildinfohelper.store_layer_info(event) + elif event.type == "BuildStatsList": + buildinfohelper.store_tasks_stats(event) + elif event.type == "ImagePkgList": + buildinfohelper.store_target_package_data(event) + elif event.type == "MissedSstate": + buildinfohelper.store_missed_state_tasks(event) + elif event.type == "SDKArtifactInfo": + buildinfohelper.scan_sdk_artifacts(event) + elif event.type == "SetBRBE": + buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) + elif event.type == "TaskArtifacts": + buildinfohelper.scan_task_artifacts(event) + elif event.type == "OSErrorException": + logger.error(event) + else: + logger.error("Unprocessed MetadataEvent %s", event.type) + continue + + if isinstance(event, bb.cooker.CookerExit): + # shutdown when bitbake server shuts down + main.shutdown = 1 + continue + + if isinstance(event, bb.event.DepTreeGenerated): + buildinfohelper.store_dependency_information(event) + continue + + logger.warning("Unknown event: %s", event) + return_value += 1 + + except EnvironmentError as ioerror: + logger.warning("EnvironmentError: %s" % ioerror) + # ignore interrupted io system calls + if ioerror.args[0] == 4: # errno 4 is EINTR + logger.warning("Skipped EINTR: %s" % ioerror) + else: + raise + except KeyboardInterrupt: + if params.observe_only: + print("\nKeyboard Interrupt, exiting observer...") + main.shutdown = 2 + if not params.observe_only and main.shutdown == 1: + print("\nSecond Keyboard Interrupt, stopping...\n") + _, error = server.runCommand(["stateForceShutdown"]) + if error: + logger.error("Unable to cleanly stop: %s" % error) + if not params.observe_only and main.shutdown == 0: + print("\nKeyboard Interrupt, closing down...\n") + interrupted = True + _, error = server.runCommand(["stateShutdown"]) + if error: + logger.error("Unable to cleanly shutdown: %s" % error) + buildinfohelper.cancel_cli_build() + main.shutdown = main.shutdown + 1 + except Exception as e: + # print errors to log + import traceback + from pprint import pformat + exception_data = traceback.format_exc() + logger.error("%s\n%s" , e, exception_data) + + # save them to database, if possible; if it fails, we already logged to console. + try: + buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) + except Exception as ce: + logger.error("CRITICAL - Failed to to save toaster exception to the database: %s", str(ce)) + + # make sure we return with an error + return_value += 1 + + if interrupted and return_value == 0: + return_value += 1 + + logger.warning("Return value is %d", return_value) + return return_value diff --git a/poky/bitbake/lib/bb/ui/uievent.py b/poky/bitbake/lib/bb/ui/uievent.py new file mode 100644 index 000000000..9542b911c --- /dev/null +++ b/poky/bitbake/lib/bb/ui/uievent.py @@ -0,0 +1,161 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +""" +Use this class to fork off a thread to recieve event callbacks from the bitbake +server and queue them for the UI to process. This process must be used to avoid +client/server deadlocks. +""" + +import socket, threading, pickle, collections +from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler + +class BBUIEventQueue: + def __init__(self, BBServer, clientinfo=("localhost, 0")): + + self.eventQueue = [] + self.eventQueueLock = threading.Lock() + self.eventQueueNotify = threading.Event() + + self.BBServer = BBServer + self.clientinfo = clientinfo + + server = UIXMLRPCServer(self.clientinfo) + self.host, self.port = server.socket.getsockname() + + server.register_function( self.system_quit, "event.quit" ) + server.register_function( self.send_event, "event.sendpickle" ) + server.socket.settimeout(1) + + self.EventHandle = None + + # the event handler registration may fail here due to cooker being in invalid state + # this is a transient situation, and we should retry a couple of times before + # giving up + + for count_tries in range(5): + ret = self.BBServer.registerEventHandler(self.host, self.port) + + if isinstance(ret, collections.Iterable): + self.EventHandle, error = ret + else: + self.EventHandle = ret + error = "" + + if self.EventHandle != None: + break + + errmsg = "Could not register UI event handler. Error: %s, host %s, "\ + "port %d" % (error, self.host, self.port) + bb.warn("%s, retry" % errmsg) + + import time + time.sleep(1) + else: + raise Exception(errmsg) + + self.server = server + + self.t = threading.Thread() + self.t.setDaemon(True) + self.t.run = self.startCallbackHandler + self.t.start() + + def getEvent(self): + + self.eventQueueLock.acquire() + + if len(self.eventQueue) == 0: + self.eventQueueLock.release() + return None + + item = self.eventQueue.pop(0) + + if len(self.eventQueue) == 0: + self.eventQueueNotify.clear() + + self.eventQueueLock.release() + return item + + def waitEvent(self, delay): + self.eventQueueNotify.wait(delay) + return self.getEvent() + + def queue_event(self, event): + self.eventQueueLock.acquire() + self.eventQueue.append(event) + self.eventQueueNotify.set() + self.eventQueueLock.release() + + def send_event(self, event): + self.queue_event(pickle.loads(event)) + + def startCallbackHandler(self): + + self.server.timeout = 1 + bb.utils.set_process_name("UIEventQueue") + while not self.server.quit: + try: + self.server.handle_request() + except Exception as e: + import traceback + logger.error("BBUIEventQueue.startCallbackHandler: Exception while trying to handle request: %s\n%s" % (e, traceback.format_exc())) + + self.server.server_close() + + def system_quit( self ): + """ + Shut down the callback thread + """ + try: + self.BBServer.unregisterEventHandler(self.EventHandle) + except: + pass + self.server.quit = True + +class UIXMLRPCServer (SimpleXMLRPCServer): + + def __init__( self, interface ): + self.quit = False + SimpleXMLRPCServer.__init__( self, + interface, + requestHandler=SimpleXMLRPCRequestHandler, + logRequests=False, allow_none=True, use_builtin_types=True) + + def get_request(self): + while not self.quit: + try: + sock, addr = self.socket.accept() + sock.settimeout(1) + return (sock, addr) + except socket.timeout: + pass + return (None, None) + + def close_request(self, request): + if request is None: + return + SimpleXMLRPCServer.close_request(self, request) + + def process_request(self, request, client_address): + if request is None: + return + SimpleXMLRPCServer.process_request(self, request, client_address) + diff --git a/poky/bitbake/lib/bb/ui/uihelper.py b/poky/bitbake/lib/bb/ui/uihelper.py new file mode 100644 index 000000000..963c1ea2d --- /dev/null +++ b/poky/bitbake/lib/bb/ui/uihelper.py @@ -0,0 +1,70 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import bb.build +import time + +class BBUIHelper: + def __init__(self): + self.needUpdate = False + self.running_tasks = {} + # Running PIDs preserves the order tasks were executed in + self.running_pids = [] + self.failed_tasks = [] + self.tasknumber_current = 0 + self.tasknumber_total = 0 + + def eventHandler(self, event): + if isinstance(event, bb.build.TaskStarted): + if event._mc != "default": + self.running_tasks[event.pid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time() } + else: + self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() } + self.running_pids.append(event.pid) + self.needUpdate = True + elif isinstance(event, bb.build.TaskSucceeded): + del self.running_tasks[event.pid] + self.running_pids.remove(event.pid) + self.needUpdate = True + elif isinstance(event, bb.build.TaskFailedSilent): + del self.running_tasks[event.pid] + self.running_pids.remove(event.pid) + # Don't add to the failed tasks list since this is e.g. a setscene task failure + self.needUpdate = True + elif isinstance(event, bb.build.TaskFailed): + del self.running_tasks[event.pid] + self.running_pids.remove(event.pid) + self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)}) + self.needUpdate = True + elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1 + self.tasknumber_total = event.stats.total + self.needUpdate = True + elif isinstance(event, bb.build.TaskProgress): + if event.pid > 0: + self.running_tasks[event.pid]['progress'] = event.progress + self.running_tasks[event.pid]['rate'] = event.rate + self.needUpdate = True + else: + return False + return True + + def getTasks(self): + self.needUpdate = False + return (self.running_tasks, self.failed_tasks) |