lib/bb/taskdata.py

   1 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 """
   5 BitBake 'TaskData' implementation
   6
   7 Task data collection and handling
   8
   9 """
  10
  11 # Copyright (C) 2006  Richard Purdie
  12 #
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License version 2 as
  15 # published by the Free Software Foundation.
  16 #
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU General Public License along
  23 # with this program; if not, write to the Free Software Foundation, Inc.,
  24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  25
  26 from bb import data, fetch, event, mkdirhier, utils
  27 import bb, os
  28
  29 class TaskData:
  30     """
  31     BitBake Task Data implementation
  32     """
  33     def __init__(self, abort = True):
  34         self.build_names_index = []
  35         self.run_names_index = []
  36         self.fn_index = []
  37
  38         self.build_targets = {}
  39         self.run_targets = {}
  40
  41         self.external_targets = []
  42
  43         self.tasks_fnid = []
  44         self.tasks_name = []
  45         self.tasks_tdepends = []
  46         self.tasks_idepends = []
  47         # Cache to speed up task ID lookups
  48         self.tasks_lookup = {}
  49
  50         self.depids = {}
  51         self.rdepids = {}
  52
  53         self.consider_msgs_cache = []
  54
  55         self.failed_deps = []
  56         self.failed_rdeps = []
  57         self.failed_fnids = []
  58
  59         self.abort = abort
  60
  61     def getbuild_id(self, name):
  62         """
  63         Return an ID number for the build target name.
  64         If it doesn't exist, create one.
  65         """
  66         if not name in self.build_names_index:
  67             self.build_names_index.append(name)
  68             return len(self.build_names_index) - 1
  69
  70         return self.build_names_index.index(name)
  71
  72     def getrun_id(self, name):
  73         """
  74         Return an ID number for the run target name.
  75         If it doesn't exist, create one.
  76         """
  77         if not name in self.run_names_index:
  78             self.run_names_index.append(name)
  79             return len(self.run_names_index) - 1
  80
  81         return self.run_names_index.index(name)
  82
  83     def getfn_id(self, name):
  84         """
  85         Return an ID number for the filename.
  86         If it doesn't exist, create one.
  87         """
  88         if not name in self.fn_index:
  89             self.fn_index.append(name)
  90             return len(self.fn_index) - 1
  91
  92         return self.fn_index.index(name)
  93
  94     def gettask_id(self, fn, task, create = True):
  95         """
  96         Return an ID number for the task matching fn and task.
  97         If it doesn't exist, create one by default.
  98         Optionally return None instead.
  99         """
 100         fnid = self.getfn_id(fn)
 101
 102         if fnid in self.tasks_lookup:
 103             if task in self.tasks_lookup[fnid]:
 104                 return self.tasks_lookup[fnid][task]
 105
 106         if not create:
 107             return None
 108
 109         self.tasks_name.append(task)
 110         self.tasks_fnid.append(fnid)
 111         self.tasks_tdepends.append([])
 112         self.tasks_idepends.append([])
 113
 114         listid = len(self.tasks_name) - 1
 115
 116         if fnid not in self.tasks_lookup:
 117             self.tasks_lookup[fnid] = {}
 118         self.tasks_lookup[fnid][task] = listid
 119
 120         return listid
 121
 122     def add_tasks(self, fn, dataCache):
 123         """
 124         Add tasks for a given fn to the database
 125         """
 126
 127         task_graph = dataCache.task_queues[fn]
 128         task_deps = dataCache.task_deps[fn]
 129
 130         fnid = self.getfn_id(fn)
 131
 132         if fnid in self.failed_fnids:
 133             bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
 134
 135         # Check if we've already seen this fn
 136         if fnid in self.tasks_fnid:
 137             return
 138
 139         for task in task_graph.allnodes():
 140
 141             # Work out task dependencies
 142             parentids = []
 143             for dep in task_graph.getparents(task):
 144                 parentid = self.gettask_id(fn, dep)
 145                 parentids.append(parentid)
 146             taskid = self.gettask_id(fn, task)
 147             self.tasks_tdepends[taskid].extend(parentids)
 148
 149             # Touch all intertask dependencies
 150             if 'depends' in task_deps and task in task_deps['depends']:
 151                 ids = []
 152                 for dep in task_deps['depends'][task].split(" "):
 153                     if dep:
 154                         ids.append(str(self.getbuild_id(dep.split(":")[0])) + ":" + dep.split(":")[1])
 155                 self.tasks_idepends[taskid].extend(ids)
 156
 157         # Work out build dependencies
 158         if not fnid in self.depids:
 159             dependids = {}
 160             for depend in dataCache.deps[fn]:
 161                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
 162                 dependids[self.getbuild_id(depend)] = None
 163             self.depids[fnid] = dependids.keys()
 164
 165         # Work out runtime dependencies
 166         if not fnid in self.rdepids:
 167             rdependids = {}
 168             rdepends = dataCache.rundeps[fn]
 169             rrecs = dataCache.runrecs[fn]
 170             for package in rdepends:
 171                 for rdepend in rdepends[package]:
 172                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
 173                     rdependids[self.getrun_id(rdepend)] = None
 174             for package in rrecs:
 175                 for rdepend in rrecs[package]:
 176                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
 177                     rdependids[self.getrun_id(rdepend)] = None
 178             self.rdepids[fnid] = rdependids.keys()
 179
 180         for dep in self.depids[fnid]:
 181             if dep in self.failed_deps:
 182                 self.fail_fnid(fnid)
 183                 return
 184         for dep in self.rdepids[fnid]:
 185             if dep in self.failed_rdeps:
 186                 self.fail_fnid(fnid)
 187                 return
 188
 189     def have_build_target(self, target):
 190         """
 191         Have we a build target matching this name?
 192         """
 193         targetid = self.getbuild_id(target)
 194
 195         if targetid in self.build_targets:
 196             return True
 197         return False
 198
 199     def have_runtime_target(self, target):
 200         """
 201         Have we a runtime target matching this name?
 202         """
 203         targetid = self.getrun_id(target)
 204
 205         if targetid in self.run_targets:
 206             return True
 207         return False
 208
 209     def add_build_target(self, fn, item):
 210         """
 211         Add a build target.
 212         If already present, append the provider fn to the list
 213         """
 214         targetid = self.getbuild_id(item)
 215         fnid = self.getfn_id(fn)
 216
 217         if targetid in self.build_targets:
 218             if fnid in self.build_targets[targetid]:
 219                 return
 220             self.build_targets[targetid].append(fnid)
 221             return
 222         self.build_targets[targetid] = [fnid]
 223
 224     def add_runtime_target(self, fn, item):
 225         """
 226         Add a runtime target.
 227         If already present, append the provider fn to the list
 228         """
 229         targetid = self.getrun_id(item)
 230         fnid = self.getfn_id(fn)
 231
 232         if targetid in self.run_targets:
 233             if fnid in self.run_targets[targetid]:
 234                 return
 235             self.run_targets[targetid].append(fnid)
 236             return
 237         self.run_targets[targetid] = [fnid]
 238
 239     def mark_external_target(self, item):
 240         """
 241         Mark a build target as being externally requested
 242         """
 243         targetid = self.getbuild_id(item)
 244
 245         if targetid not in self.external_targets:
 246             self.external_targets.append(targetid)
 247
 248     def get_unresolved_build_targets(self, dataCache):
 249         """
 250         Return a list of build targets who's providers
 251         are unknown.
 252         """
 253         unresolved = []
 254         for target in self.build_names_index:
 255             if target in dataCache.ignored_dependencies:
 256                 continue
 257             if self.build_names_index.index(target) in self.failed_deps:
 258                 continue
 259             if not self.have_build_target(target):
 260                 unresolved.append(target)
 261         return unresolved
 262
 263     def get_unresolved_run_targets(self, dataCache):
 264         """
 265         Return a list of runtime targets who's providers
 266         are unknown.
 267         """
 268         unresolved = []
 269         for target in self.run_names_index:
 270             if target in dataCache.ignored_dependencies:
 271                 continue
 272             if self.run_names_index.index(target) in self.failed_rdeps:
 273                 continue
 274             if not self.have_runtime_target(target):
 275                 unresolved.append(target)
 276         return unresolved
 277
 278     def get_provider(self, item):
 279         """
 280         Return a list of providers of item
 281         """
 282         targetid = self.getbuild_id(item)
 283
 284         return self.build_targets[targetid]
 285
 286     def get_dependees(self, itemid):
 287         """
 288         Return a list of targets which depend on item
 289         """
 290         dependees = []
 291         for fnid in self.depids:
 292             if itemid in self.depids[fnid]:
 293                 dependees.append(fnid)
 294         return dependees
 295
 296     def get_dependees_str(self, item):
 297         """
 298         Return a list of targets which depend on item as a user readable string
 299         """
 300         itemid = self.getbuild_id(item)
 301         dependees = []
 302         for fnid in self.depids:
 303             if itemid in self.depids[fnid]:
 304                 dependees.append(self.fn_index[fnid])
 305         return dependees
 306
 307     def get_rdependees(self, itemid):
 308         """
 309         Return a list of targets which depend on runtime item
 310         """
 311         dependees = []
 312         for fnid in self.rdepids:
 313             if itemid in self.rdepids[fnid]:
 314                 dependees.append(fnid)
 315         return dependees
 316
 317     def get_rdependees_str(self, item):
 318         """
 319         Return a list of targets which depend on runtime item as a user readable string
 320         """
 321         itemid = self.getrun_id(item)
 322         dependees = []
 323         for fnid in self.rdepids:
 324             if itemid in self.rdepids[fnid]:
 325                 dependees.append(self.fn_index[fnid])
 326         return dependees
 327
 328     def add_provider(self, cfgData, dataCache, item):
 329         try:
 330             self.add_provider_internal(cfgData, dataCache, item)
 331         except bb.providers.NoProvider:
 332             if self.abort:
 333                 bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
 334                 raise
 335             targetid = self.getbuild_id(item)
 336             self.remove_buildtarget(targetid)
 337
 338         self.mark_external_target(item)
 339
 340     def add_provider_internal(self, cfgData, dataCache, item):
 341         """
 342         Add the providers of item to the task data
 343         Mark entries were specifically added externally as against dependencies
 344         added internally during dependency resolution
 345         """
 346
 347         if item in dataCache.ignored_dependencies:
 348             return
 349
 350         if not item in dataCache.providers:
 351             bb.msg.note(2, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
 352             bb.event.fire(bb.event.NoProvider(item, cfgData))
 353             raise bb.providers.NoProvider(item)
 354
 355         if self.have_build_target(item):
 356             return
 357
 358         all_p = dataCache.providers[item]
 359
 360         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 361
 362         for p in eligible:
 363             fnid = self.getfn_id(p)
 364             if fnid in self.failed_fnids:
 365                 eligible.remove(p)
 366
 367         if not eligible:
 368             bb.msg.note(2, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
 369             bb.event.fire(bb.event.NoProvider(item, cfgData))
 370             raise bb.providers.NoProvider(item)
 371
 372         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
 373         if prefervar:
 374             dataCache.preferred[item] = prefervar
 375
 376         discriminated = False
 377         if item in dataCache.preferred:
 378             for p in eligible:
 379                 pn = dataCache.pkg_fn[p]
 380                 if dataCache.preferred[item] == pn:
 381                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
 382                     eligible.remove(p)
 383                     eligible = [p] + eligible
 384                     discriminated = True
 385                     break
 386
 387         if len(eligible) > 1 and discriminated == False:
 388             if item not in self.consider_msgs_cache:
 389                 providers_list = []
 390                 for fn in eligible:
 391                     providers_list.append(dataCache.pkg_fn[fn])
 392                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
 393                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
 394                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
 395             self.consider_msgs_cache.append(item)
 396
 397         for fn in eligible:
 398             fnid = self.getfn_id(fn)
 399             if fnid in self.failed_fnids:
 400                 continue
 401             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
 402             self.add_build_target(fn, item)
 403             self.add_tasks(fn, dataCache)
 404
 405
 406             #item = dataCache.pkg_fn[fn]
 407
 408     def add_rprovider(self, cfgData, dataCache, item):
 409         """
 410         Add the runtime providers of item to the task data
 411         (takes item names from RDEPENDS/PACKAGES namespace)
 412         """
 413
 414         if item in dataCache.ignored_dependencies:
 415             return
 416
 417         if self.have_runtime_target(item):
 418             return
 419
 420         all_p = bb.providers.getRuntimeProviders(dataCache, item)
 421
 422         if not all_p:
 423             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
 424             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 425             raise bb.providers.NoRProvider(item)
 426
 427         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 428
 429         for p in eligible:
 430             fnid = self.getfn_id(p)
 431             if fnid in self.failed_fnids:
 432                 eligible.remove(p)
 433
 434         if not eligible:
 435             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
 436             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 437             raise bb.providers.NoRProvider(item)
 438
 439         # Should use dataCache.preferred here?
 440         preferred = []
 441         for p in eligible:
 442             pn = dataCache.pkg_fn[p]
 443             provides = dataCache.pn_provides[pn]
 444             for provide in provides:
 445                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
 446                 if prefervar == pn:
 447                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
 448                     eligible.remove(p)
 449                     eligible = [p] + eligible
 450                     preferred.append(p)
 451                     break
 452
 453         if len(eligible) > 1 and len(preferred) == 0:
 454             if item not in self.consider_msgs_cache:
 455                 providers_list = []
 456                 for fn in eligible:
 457                     providers_list.append(dataCache.pkg_fn[fn])
 458                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
 459                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
 460                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 461             self.consider_msgs_cache.append(item)
 462
 463         if len(preferred) > 1:
 464             if item not in self.consider_msgs_cache:
 465                 providers_list = []
 466                 for fn in preferred:
 467                     providers_list.append(dataCache.pkg_fn[fn])
 468                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
 469                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
 470                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 471             self.consider_msgs_cache.append(item)
 472
 473         # run through the list until we find one that we can build
 474         for fn in eligible:
 475             fnid = self.getfn_id(fn)
 476             if fnid in self.failed_fnids:
 477                 continue
 478             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
 479             self.add_runtime_target(fn, item)
 480             self.add_tasks(fn, dataCache)
 481
 482     def fail_fnid(self, fnid):
 483         """
 484         Mark a file as failed (unbuildable)
 485         Remove any references from build and runtime provider lists
 486         """
 487         if fnid in self.failed_fnids:
 488             return
 489         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
 490         self.failed_fnids.append(fnid)
 491         for target in self.build_targets:
 492             if fnid in self.build_targets[target]:
 493                 self.build_targets[target].remove(fnid)
 494                 if len(self.build_targets[target]) == 0:
 495                     self.remove_buildtarget(target)
 496         for target in self.run_targets:
 497             if fnid in self.run_targets[target]:
 498                 self.run_targets[target].remove(fnid)
 499                 if len(self.run_targets[target]) == 0:
 500                     self.remove_runtarget(target)
 501
 502     def remove_buildtarget(self, targetid):
 503         """
 504         Mark a build target as failed (unbuildable)
 505         Trigger removal of any files that have this as a dependency
 506         """
 507         bb.msg.note(2, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
 508         self.failed_deps.append(targetid)
 509         dependees = self.get_dependees(targetid)
 510         for fnid in dependees:
 511             self.fail_fnid(fnid)
 512         if self.abort and targetid in self.external_targets:
 513             bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s" % self.build_names_index[targetid])
 514             raise bb.providers.NoProvider
 515
 516     def remove_runtarget(self, targetid):
 517         """
 518         Mark a run target as failed (unbuildable)
 519         Trigger removal of any files that have this as a dependency
 520         """
 521         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
 522         self.failed_rdeps.append(targetid)
 523         dependees = self.get_rdependees(targetid)
 524         for fnid in dependees:
 525             self.fail_fnid(fnid)
 526
 527     def add_unresolved(self, cfgData, dataCache):
 528         """
 529         Resolve all unresolved build and runtime targets
 530         """
 531         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
 532         while 1:
 533             added = 0
 534             for target in self.get_unresolved_build_targets(dataCache):
 535                 try:
 536                     self.add_provider_internal(cfgData, dataCache, target)
 537                     added = added + 1
 538                 except bb.providers.NoProvider:
 539                     targetid = self.getbuild_id(target)
 540                     if self.abort and targetid in self.external_targets:
 541                         raise
 542                     self.remove_buildtarget(targetid)
 543             for target in self.get_unresolved_run_targets(dataCache):
 544                 try:
 545                     self.add_rprovider(cfgData, dataCache, target)
 546                     added = added + 1
 547                 except bb.providers.NoRProvider:
 548                     self.remove_runtarget(self.getrun_id(target))
 549             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
 550             if added == 0:
 551                 break
 552         # self.dump_data()
 553
 554     def dump_data(self):
 555         """
 556         Dump some debug information on the internal data structures
 557         """
 558         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
 559         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index))
 560         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
 561         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index))
 562         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
 563         for target in self.build_targets.keys():
 564             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
 565         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
 566         for target in self.run_targets.keys():
 567             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
 568         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
 569         for task in range(len(self.tasks_name)):
 570             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
 571                 task,
 572                 self.fn_index[self.tasks_fnid[task]],
 573                 self.tasks_name[task],
 574                 self.tasks_tdepends[task]))
 575         bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):")
 576         for fnid in self.rdepids:
 577             bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))
 578
 579