taskdata: Remove supriours PREFERRED_PROVIDER warnings
[bitbake.git] / lib / bb / taskdata.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 """
5 BitBake 'TaskData' implementation
6
7 Task data collection and handling
8
9 """
10
11 # Copyright (C) 2006  Richard Purdie
12 #
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License version 2 as
15 # published by the Free Software Foundation.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License along
23 # with this program; if not, write to the Free Software Foundation, Inc.,
24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25
26 from bb import data, fetch, event, mkdirhier, utils
27 import bb, os
28
29 class TaskData:
30     """
31     BitBake Task Data implementation
32     """
33     def __init__(self, abort = True):
34         self.build_names_index = []
35         self.run_names_index = []
36         self.fn_index = []
37
38         self.build_targets = {}
39         self.run_targets = {}
40
41         self.external_targets = []
42
43         self.tasks_fnid = []
44         self.tasks_name = []
45         self.tasks_tdepends = []
46         self.tasks_idepends = []
47         # Cache to speed up task ID lookups
48         self.tasks_lookup = {}
49
50         self.depids = {}
51         self.rdepids = {}
52
53         self.consider_msgs_cache = []
54
55         self.failed_deps = []
56         self.failed_rdeps = []
57         self.failed_fnids = []
58
59         self.abort = abort
60
61     def getbuild_id(self, name):
62         """
63         Return an ID number for the build target name.
64         If it doesn't exist, create one.
65         """
66         if not name in self.build_names_index:
67             self.build_names_index.append(name)
68             return len(self.build_names_index) - 1
69
70         return self.build_names_index.index(name)
71
72     def getrun_id(self, name):
73         """
74         Return an ID number for the run target name. 
75         If it doesn't exist, create one.
76         """
77         if not name in self.run_names_index:
78             self.run_names_index.append(name)
79             return len(self.run_names_index) - 1
80
81         return self.run_names_index.index(name)
82
83     def getfn_id(self, name):
84         """
85         Return an ID number for the filename. 
86         If it doesn't exist, create one.
87         """
88         if not name in self.fn_index:
89             self.fn_index.append(name)
90             return len(self.fn_index) - 1
91
92         return self.fn_index.index(name)
93
94     def gettask_id(self, fn, task, create = True):
95         """
96         Return an ID number for the task matching fn and task.
97         If it doesn't exist, create one by default.
98         Optionally return None instead.
99         """
100         fnid = self.getfn_id(fn)
101
102         if fnid in self.tasks_lookup:
103             if task in self.tasks_lookup[fnid]:
104                 return self.tasks_lookup[fnid][task]
105
106         if not create:
107             return None
108
109         self.tasks_name.append(task)
110         self.tasks_fnid.append(fnid)
111         self.tasks_tdepends.append([])
112         self.tasks_idepends.append([])
113
114         listid = len(self.tasks_name) - 1
115
116         if fnid not in self.tasks_lookup:
117             self.tasks_lookup[fnid] = {}
118         self.tasks_lookup[fnid][task] = listid
119
120         return listid
121
122     def add_tasks(self, fn, dataCache):
123         """
124         Add tasks for a given fn to the database
125         """
126
127         task_graph = dataCache.task_queues[fn]
128         task_deps = dataCache.task_deps[fn]
129
130         fnid = self.getfn_id(fn)
131
132         if fnid in self.failed_fnids:
133             bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
134
135         # Check if we've already seen this fn
136         if fnid in self.tasks_fnid:
137             return
138
139         for task in task_graph.allnodes():
140
141             # Work out task dependencies
142             parentids = []
143             for dep in task_graph.getparents(task):
144                 parentid = self.gettask_id(fn, dep)
145                 parentids.append(parentid)
146             taskid = self.gettask_id(fn, task)
147             self.tasks_tdepends[taskid].extend(parentids)
148
149             # Touch all intertask dependencies
150             if 'depends' in task_deps and task in task_deps['depends']:
151                 ids = []
152                 for dep in task_deps['depends'][task].split(" "):
153                     if dep:
154                         ids.append(str(self.getbuild_id(dep.split(":")[0])) + ":" + dep.split(":")[1])
155                 self.tasks_idepends[taskid].extend(ids)
156
157         # Work out build dependencies
158         if not fnid in self.depids:
159             dependids = {}
160             for depend in dataCache.deps[fn]:
161                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
162                 dependids[self.getbuild_id(depend)] = None
163             self.depids[fnid] = dependids.keys()
164
165         # Work out runtime dependencies
166         if not fnid in self.rdepids:
167             rdependids = {}
168             rdepends = dataCache.rundeps[fn]
169             rrecs = dataCache.runrecs[fn]
170             for package in rdepends:
171                 for rdepend in rdepends[package]:
172                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
173                     rdependids[self.getrun_id(rdepend)] = None
174             for package in rrecs:
175                 for rdepend in rrecs[package]:
176                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
177                     rdependids[self.getrun_id(rdepend)] = None
178             self.rdepids[fnid] = rdependids.keys()
179
180         for dep in self.depids[fnid]:
181             if dep in self.failed_deps:
182                 self.fail_fnid(fnid)
183                 return
184         for dep in self.rdepids[fnid]:
185             if dep in self.failed_rdeps:
186                 self.fail_fnid(fnid)
187                 return
188
189     def have_build_target(self, target):
190         """
191         Have we a build target matching this name?
192         """
193         targetid = self.getbuild_id(target)
194
195         if targetid in self.build_targets:
196             return True
197         return False
198
199     def have_runtime_target(self, target):
200         """
201         Have we a runtime target matching this name?
202         """
203         targetid = self.getrun_id(target)
204
205         if targetid in self.run_targets:
206             return True
207         return False
208
209     def add_build_target(self, fn, item):
210         """
211         Add a build target.
212         If already present, append the provider fn to the list
213         """
214         targetid = self.getbuild_id(item)
215         fnid = self.getfn_id(fn)
216
217         if targetid in self.build_targets:
218             if fnid in self.build_targets[targetid]:
219                 return
220             self.build_targets[targetid].append(fnid)
221             return
222         self.build_targets[targetid] = [fnid]
223
224     def add_runtime_target(self, fn, item):
225         """
226         Add a runtime target.
227         If already present, append the provider fn to the list
228         """
229         targetid = self.getrun_id(item)
230         fnid = self.getfn_id(fn)
231
232         if targetid in self.run_targets:
233             if fnid in self.run_targets[targetid]:
234                 return
235             self.run_targets[targetid].append(fnid)
236             return
237         self.run_targets[targetid] = [fnid]
238
239     def mark_external_target(self, item):
240         """
241         Mark a build target as being externally requested
242         """
243         targetid = self.getbuild_id(item)
244
245         if targetid not in self.external_targets:
246             self.external_targets.append(targetid)
247
248     def get_unresolved_build_targets(self, dataCache):
249         """
250         Return a list of build targets who's providers 
251         are unknown.
252         """
253         unresolved = []
254         for target in self.build_names_index:
255             if target in dataCache.ignored_dependencies:
256                 continue
257             if self.build_names_index.index(target) in self.failed_deps:
258                 continue
259             if not self.have_build_target(target):
260                 unresolved.append(target)
261         return unresolved
262
263     def get_unresolved_run_targets(self, dataCache):
264         """
265         Return a list of runtime targets who's providers 
266         are unknown.
267         """
268         unresolved = []
269         for target in self.run_names_index:
270             if target in dataCache.ignored_dependencies:
271                 continue
272             if self.run_names_index.index(target) in self.failed_rdeps:
273                 continue
274             if not self.have_runtime_target(target):
275                 unresolved.append(target)
276         return unresolved
277
278     def get_provider(self, item):
279         """
280         Return a list of providers of item
281         """
282         targetid = self.getbuild_id(item)
283    
284         return self.build_targets[targetid]
285
286     def get_dependees(self, itemid):
287         """
288         Return a list of targets which depend on item
289         """
290         dependees = []
291         for fnid in self.depids:
292             if itemid in self.depids[fnid]:
293                 dependees.append(fnid)
294         return dependees
295
296     def get_dependees_str(self, item):
297         """
298         Return a list of targets which depend on item as a user readable string
299         """
300         itemid = self.getbuild_id(item)
301         dependees = []
302         for fnid in self.depids:
303             if itemid in self.depids[fnid]:
304                 dependees.append(self.fn_index[fnid])
305         return dependees
306
307     def get_rdependees(self, itemid):
308         """
309         Return a list of targets which depend on runtime item
310         """
311         dependees = []
312         for fnid in self.rdepids:
313             if itemid in self.rdepids[fnid]:
314                 dependees.append(fnid)
315         return dependees
316
317     def get_rdependees_str(self, item):
318         """
319         Return a list of targets which depend on runtime item as a user readable string
320         """
321         itemid = self.getrun_id(item)
322         dependees = []
323         for fnid in self.rdepids:
324             if itemid in self.rdepids[fnid]:
325                 dependees.append(self.fn_index[fnid])
326         return dependees
327
328     def add_provider(self, cfgData, dataCache, item):
329         try:
330             self.add_provider_internal(cfgData, dataCache, item)
331         except bb.providers.NoProvider:
332             if self.abort:
333                 bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
334                 raise
335             targetid = self.getbuild_id(item)
336             self.remove_buildtarget(targetid)
337
338         self.mark_external_target(item)
339
340     def add_provider_internal(self, cfgData, dataCache, item):
341         """
342         Add the providers of item to the task data
343         Mark entries were specifically added externally as against dependencies 
344         added internally during dependency resolution
345         """
346
347         if item in dataCache.ignored_dependencies:
348             return
349
350         if not item in dataCache.providers:
351             bb.msg.note(2, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
352             bb.event.fire(bb.event.NoProvider(item, cfgData))
353             raise bb.providers.NoProvider(item)
354
355         if self.have_build_target(item):
356             return
357
358         all_p = dataCache.providers[item]
359
360         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
361
362         for p in eligible:
363             fnid = self.getfn_id(p)
364             if fnid in self.failed_fnids:
365                 eligible.remove(p)
366
367         if not eligible:
368             bb.msg.note(2, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
369             bb.event.fire(bb.event.NoProvider(item, cfgData))
370             raise bb.providers.NoProvider(item)
371
372         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
373         if prefervar:
374             dataCache.preferred[item] = prefervar
375
376         discriminated = False
377         if item in dataCache.preferred:
378             for p in eligible:
379                 pn = dataCache.pkg_fn[p]
380                 if dataCache.preferred[item] == pn:
381                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
382                     eligible.remove(p)
383                     eligible = [p] + eligible
384                     discriminated = True
385                     break
386
387         if len(eligible) > 1 and discriminated == False:
388             if item not in self.consider_msgs_cache:
389                 providers_list = []
390                 for fn in eligible:
391                     providers_list.append(dataCache.pkg_fn[fn])
392                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
393                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
394                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
395             self.consider_msgs_cache.append(item)
396
397         for fn in eligible:
398             fnid = self.getfn_id(fn)
399             if fnid in self.failed_fnids:
400                 continue
401             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
402             self.add_build_target(fn, item)
403             self.add_tasks(fn, dataCache)
404
405
406             #item = dataCache.pkg_fn[fn]
407
408     def add_rprovider(self, cfgData, dataCache, item):
409         """
410         Add the runtime providers of item to the task data
411         (takes item names from RDEPENDS/PACKAGES namespace)
412         """
413
414         if item in dataCache.ignored_dependencies:
415             return
416
417         if self.have_runtime_target(item):
418             return
419
420         all_p = bb.providers.getRuntimeProviders(dataCache, item)
421
422         if not all_p:
423             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
424             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
425             raise bb.providers.NoRProvider(item)
426
427         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
428
429         for p in eligible:
430             fnid = self.getfn_id(p)
431             if fnid in self.failed_fnids:
432                 eligible.remove(p)
433
434         if not eligible:
435             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
436             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
437             raise bb.providers.NoRProvider(item)
438
439         # Should use dataCache.preferred here?
440         preferred = []
441         for p in eligible:
442             pn = dataCache.pkg_fn[p]
443             provides = dataCache.pn_provides[pn]
444             for provide in provides:
445                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
446                 if prefervar == pn:
447                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
448                     eligible.remove(p)
449                     eligible = [p] + eligible
450                     preferred.append(p)
451                     break
452
453         if len(eligible) > 1 and len(preferred) == 0:
454             if item not in self.consider_msgs_cache:
455                 providers_list = []
456                 for fn in eligible:
457                     providers_list.append(dataCache.pkg_fn[fn])
458                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
459                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
460                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
461             self.consider_msgs_cache.append(item)
462
463         if len(preferred) > 1:
464             if item not in self.consider_msgs_cache:
465                 providers_list = []
466                 for fn in preferred:
467                     providers_list.append(dataCache.pkg_fn[fn])
468                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
469                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
470                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
471             self.consider_msgs_cache.append(item)
472
473         # run through the list until we find one that we can build
474         for fn in eligible:
475             fnid = self.getfn_id(fn)
476             if fnid in self.failed_fnids:
477                 continue
478             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
479             self.add_runtime_target(fn, item)
480             self.add_tasks(fn, dataCache)
481
482     def fail_fnid(self, fnid):
483         """
484         Mark a file as failed (unbuildable)
485         Remove any references from build and runtime provider lists
486         """
487         if fnid in self.failed_fnids:
488             return
489         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
490         self.failed_fnids.append(fnid)
491         for target in self.build_targets:
492             if fnid in self.build_targets[target]:
493                 self.build_targets[target].remove(fnid)
494                 if len(self.build_targets[target]) == 0:
495                     self.remove_buildtarget(target)
496         for target in self.run_targets:
497             if fnid in self.run_targets[target]:
498                 self.run_targets[target].remove(fnid)
499                 if len(self.run_targets[target]) == 0:
500                     self.remove_runtarget(target)
501
502     def remove_buildtarget(self, targetid):
503         """
504         Mark a build target as failed (unbuildable)
505         Trigger removal of any files that have this as a dependency
506         """
507         bb.msg.note(2, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
508         self.failed_deps.append(targetid)
509         dependees = self.get_dependees(targetid)
510         for fnid in dependees:
511             self.fail_fnid(fnid)
512         if self.abort and targetid in self.external_targets:
513             bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s" % self.build_names_index[targetid])
514             raise bb.providers.NoProvider
515
516     def remove_runtarget(self, targetid):
517         """
518         Mark a run target as failed (unbuildable)
519         Trigger removal of any files that have this as a dependency
520         """
521         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
522         self.failed_rdeps.append(targetid)
523         dependees = self.get_rdependees(targetid)
524         for fnid in dependees:
525             self.fail_fnid(fnid)
526
527     def add_unresolved(self, cfgData, dataCache):
528         """
529         Resolve all unresolved build and runtime targets
530         """
531         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
532         while 1:
533             added = 0
534             for target in self.get_unresolved_build_targets(dataCache):
535                 try:
536                     self.add_provider_internal(cfgData, dataCache, target)
537                     added = added + 1
538                 except bb.providers.NoProvider:
539                     targetid = self.getbuild_id(target)
540                     if self.abort and targetid in self.external_targets:
541                         raise
542                     self.remove_buildtarget(targetid)
543             for target in self.get_unresolved_run_targets(dataCache):
544                 try:
545                     self.add_rprovider(cfgData, dataCache, target)
546                     added = added + 1
547                 except bb.providers.NoRProvider:
548                     self.remove_runtarget(self.getrun_id(target))
549             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
550             if added == 0:
551                 break
552         # self.dump_data()
553
554     def dump_data(self):
555         """
556         Dump some debug information on the internal data structures
557         """
558         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
559         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index))
560         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
561         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index))
562         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
563         for target in self.build_targets.keys():
564             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
565         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
566         for target in self.run_targets.keys():
567             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
568         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
569         for task in range(len(self.tasks_name)):
570             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
571                 task, 
572                 self.fn_index[self.tasks_fnid[task]], 
573                 self.tasks_name[task], 
574                 self.tasks_tdepends[task]))
575         bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):")
576         for fnid in self.rdepids:
577             bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))
578
579