taskdata.py: Fix so only warn about failed providers once.
[bitbake.git] / lib / bb / taskdata.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 """
5 BitBake 'TaskData' implementation
6
7 Task data collection and handling
8
9 Copyright (C) 2006  Richard Purdie
10
11 This program is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License version 2 as published by the Free 
13 Software Foundation
14
15 This program is distributed in the hope that it will be useful, but WITHOUT
16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License along with
20 """
21
22 from bb import data, fetch, event, mkdirhier, utils
23 import bb, os
24
25 class TaskData:
26     """
27     BitBake Task Data implementation
28     """
29     def __init__(self):
30         self.build_names_index = []
31         self.run_names_index = []
32         self.fn_index = []
33
34         self.build_targets = {}
35         self.run_targets = {}
36
37         self.tasks_fnid = []
38         self.tasks_name = []
39         self.tasks_tdepends = []
40
41         self.depids = {}
42         self.rdepids = {}
43
44         self.consider_msgs_cache = []
45
46         self.failed_deps = []
47         self.failed_rdeps = []
48         self.failed_fnids = []
49
50
51     def matches_in_list(self, data, substring):
52         """
53         Return a list of the positions of substring in list data
54         """
55         matches = []
56         start = 0
57         while 1:
58             try:
59                 start = data.index(substring, start)
60             except ValueError:
61                 return matches
62             matches.append(start)
63             start = start + 1
64
65     def both_contain(self, list1, list2):
66         """
67         Return the items present in both list1 and list2
68         """
69         matches = []
70         for data in list1:
71             if data in list2:
72                 return data
73         return None
74
75
76     def getbuild_id(self, name):
77         """
78         Return an ID number for the build target name.
79         If it doesn't exist, create one.
80         """
81         if not name in self.build_names_index:
82             self.build_names_index.append(name)
83
84         return self.build_names_index.index(name)
85
86     def getrun_id(self, name):
87         """
88         Return an ID number for the run target name. 
89         If it doesn't exist, create one.
90         """
91         if not name in self.run_names_index:
92             self.run_names_index.append(name)
93
94         return self.run_names_index.index(name)
95
96     def getfn_id(self, name):
97         """
98         Return an ID number for the filename. 
99         If it doesn't exist, create one.
100         """
101         if not name in self.fn_index:
102             self.fn_index.append(name)
103
104         return self.fn_index.index(name)
105
106     def gettask_id(self, fn, task):
107         """
108         Return an ID number for the task matching fn and task.
109         If it doesn't exist, create one.
110         """
111         fnid = self.getfn_id(fn)
112
113         fnids = self.matches_in_list(self.tasks_fnid, fnid)
114         names = self.matches_in_list(self.tasks_name, task)
115
116         listid = self.both_contain(fnids, names)
117
118         if listid is not None:
119             return listid
120
121         self.tasks_name.append(task)
122         self.tasks_fnid.append(fnid)
123         self.tasks_tdepends.append([])
124
125         return len(self.tasks_name)-1
126
127     def add_tasks(self, fn, dataCache):
128         """
129         Add tasks for a given fn to the database
130         """
131
132         task_graph = dataCache.task_queues[fn]
133         task_deps = dataCache.task_deps[fn]
134
135         fnid = self.getfn_id(fn)
136
137         if fnid in self.failed_fnids:
138             bb.fatal("Trying to re-add a failed file? Something is broken...")
139
140         # Check if we've already seen this fn
141         if fnid in self.tasks_fnid:
142             return
143
144         # Work out task dependencies
145         for task in task_graph.allnodes():
146             parentids = []
147             for dep in task_graph.getparents(task):
148                 parentid = self.gettask_id(fn, dep)
149                 parentids.append(parentid)
150             taskid = self.gettask_id(fn, task)
151             self.tasks_tdepends[taskid].extend(parentids)
152
153         # Work out build dependencies
154         if not fnid in self.depids:
155             dependids = []
156             for depend in dataCache.deps[fn]:
157                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
158                 dependids.append(self.getbuild_id(depend))
159             self.depids[fnid] = dependids
160
161         # Work out runtime dependencies
162         if not fnid in self.rdepids:
163             rdependids = []
164             rdepends = dataCache.rundeps[fn]
165             rrecs = dataCache.runrecs[fn]
166             for package in rdepends:
167                 for rdepend in rdepends[package]:
168                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
169                     rdependids.append(self.getrun_id(rdepend))
170             for package in rrecs:
171                 for rdepend in rrecs[package]:
172                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
173                     rdependids.append(self.getrun_id(rdepend))
174             self.rdepids[fnid] = rdependids
175
176     def have_build_target(self, target):
177         """
178         Have we a build target matching this name?
179         """
180         targetid = self.getbuild_id(target)
181
182         if targetid in self.build_targets:
183             return True
184         return False
185
186     def have_runtime_target(self, target):
187         """
188         Have we a runtime target matching this name?
189         """
190         targetid = self.getrun_id(target)
191
192         if targetid in self.run_targets:
193             return True
194         return False
195
196     def add_build_target(self, fn, item):
197         """
198         Add a build target.
199         If already present, append the provider fn to the list
200         """
201         targetid = self.getbuild_id(item)
202         fnid = self.getfn_id(fn)
203
204         if targetid in self.build_targets:
205             if fnid in self.build_targets[targetid]:
206                 return
207             self.build_targets[targetid].append(fnid)
208             return
209         self.build_targets[targetid] = [fnid]
210
211     def add_runtime_target(self, fn, item):
212         """
213         Add a runtime target.
214         If already present, append the provider fn to the list
215         """
216         targetid = self.getrun_id(item)
217         fnid = self.getfn_id(fn)
218
219         if targetid in self.run_targets:
220             if fnid in self.run_targets[targetid]:
221                 return
222             self.run_targets[targetid].append(fnid)
223             return
224         self.run_targets[targetid] = [fnid]
225
226     def get_unresolved_build_targets(self, dataCache):
227         """
228         Return a list of build targets who's providers 
229         are unknown.
230         """
231         unresolved = []
232         for target in self.build_names_index:
233             if target in dataCache.ignored_dependencies:
234                 continue
235             if self.build_names_index.index(target) in self.failed_deps:
236                 continue
237             if not self.have_build_target(target):
238                 unresolved.append(target)
239         return unresolved
240
241     def get_unresolved_run_targets(self, dataCache):
242         """
243         Return a list of runtime targets who's providers 
244         are unknown.
245         """
246         unresolved = []
247         for target in self.run_names_index:
248             if target in dataCache.ignored_dependencies:
249                 continue
250             if self.run_names_index.index(target) in self.failed_rdeps:
251                 continue
252             if not self.have_runtime_target(target):
253                 unresolved.append(target)
254         return unresolved
255
256     def get_provider(self, item):
257         """
258         Return a list of providers of item
259         """
260         targetid = self.getbuild_id(item)
261    
262         return self.build_targets[targetid]
263
264     def get_dependees(self, itemid):
265         """
266         Return a list of targets which depend on item
267         """
268         dependees = []
269         for fnid in self.depids:
270             if itemid in self.depids[fnid]:
271                 dependees.append(fnid)
272         return dependees
273
274     def get_dependees_str(self, item):
275         """
276         Return a list of targets which depend on item as a user readable string
277         """
278         itemid = self.getbuild_id(item)
279         dependees = []
280         for fnid in self.depids:
281             if itemid in self.depids[fnid]:
282                 dependees.append(self.fn_index[fnid])
283         return dependees
284
285     def get_rdependees(self, itemid):
286         """
287         Return a list of targets which depend on runtime item
288         """
289         dependees = []
290         for fnid in self.rdepids:
291             if itemid in self.rdepids[fnid]:
292                 dependees.append(fnid)
293         return dependees
294
295     def get_rdependees_str(self, item):
296         """
297         Return a list of targets which depend on runtime item as a user readable string
298         """
299         itemid = self.getrun_id(item)
300         dependees = []
301         for fnid in self.rdepids:
302             if itemid in self.rdepids[fnid]:
303                 dependees.append(self.fn_index[fnid])
304         return dependees
305
306     def add_provider(self, cfgData, dataCache, item):
307         """
308         Add the providers of item to the task data
309         """
310
311         if item in dataCache.ignored_dependencies:
312             return True
313
314         if not item in dataCache.providers:
315             bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
316             bb.event.fire(bb.event.NoProvider(item, cfgData))
317             raise bb.providers.NoProvider(item)
318
319         if self.have_build_target(item):
320             return True
321
322         all_p = dataCache.providers[item]
323
324         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
325
326         for p in eligible:
327             fnid = self.getfn_id(p)
328             if fnid in self.failed_fnids:
329                 eligible.remove(p)
330
331         if not eligible:
332             bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
333             bb.event.fire(bb.event.NoProvider(item, cfgData))
334             raise bb.providers.NoProvider(item)
335
336         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
337         if prefervar:
338             dataCache.preferred[item] = prefervar
339
340         discriminated = False
341         if item in dataCache.preferred:
342             for p in eligible:
343                 pn = dataCache.pkg_fn[p]
344                 if dataCache.preferred[item] == pn:
345                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
346                     eligible.remove(p)
347                     eligible = [p] + eligible
348                     discriminated = True
349                     break
350
351         if len(eligible) > 1 and discriminated == False:
352             if item not in self.consider_msgs_cache:
353                 providers_list = []
354                 for fn in eligible:
355                     providers_list.append(dataCache.pkg_fn[fn])
356                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
357                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
358                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
359             self.consider_msgs_cache.append(item)
360
361         for fn in eligible:
362             fnid = self.getfn_id(fn)
363             if fnid in self.failed_fnids:
364                 continue
365             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
366             self.add_tasks(fn, dataCache)
367             self.add_build_target(fn, item)
368
369             item = dataCache.pkg_fn[fn]
370
371         return True
372
373     def add_rprovider(self, cfgData, dataCache, item):
374         """
375         Add the runtime providers of item to the task data
376         (takes item names from RDEPENDS/PACKAGES namespace)
377         """
378
379         if item in dataCache.ignored_dependencies:
380             return True
381
382         if self.have_runtime_target(item):
383             return True
384
385         all_p = bb.providers.getRuntimeProviders(dataCache, item)
386
387         if not all_p:
388             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
389             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
390             raise bb.providers.NoRProvider(item)
391
392         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
393
394         for p in eligible:
395             fnid = self.getfn_id(p)
396             if fnid in self.failed_fnids:
397                 eligible.remove(p)
398
399         if not eligible:
400             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
401             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
402             raise bb.providers.NoRProvider(item)
403
404         # Should use dataCache.preferred here?
405         preferred = []
406         for p in eligible:
407             pn = dataCache.pkg_fn[p]
408             provides = dataCache.pn_provides[pn]
409             for provide in provides:
410                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
411                 if prefervar == pn:
412                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
413                     eligible.remove(p)
414                     eligible = [p] + eligible
415                     preferred.append(p)
416
417         if len(eligible) > 1 and len(preferred) == 0:
418             if item not in self.consider_msgs_cache:
419                 providers_list = []
420                 for fn in eligible:
421                     providers_list.append(dataCache.pkg_fn[fn])
422                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
423                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
424                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
425             self.consider_msgs_cache.append(item)
426
427         if len(preferred) > 1:
428             if item not in self.consider_msgs_cache:
429                 providers_list = []
430                 for fn in preferred:
431                     providers_list.append(dataCache.pkg_fn[fn])
432                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available (%s);" % ", ".join(providers_list))
433                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
434                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
435             self.consider_msgs_cache.append(item)
436
437         # run through the list until we find one that we can build
438         for fn in eligible:
439             fnid = self.getfn_id(fn)
440             if fnid in self.failed_fnids:
441                 continue
442             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
443             self.add_tasks(fn, dataCache)
444             self.add_runtime_target(fn, item)
445
446         return True
447
448     def fail_fnid(self, fnid):
449         """
450         Mark a file as failed (unbuildable)
451         Remove any references from build and runtime provider lists
452         """
453         if fnid in self.failed_fnids:
454             return
455         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
456         self.failed_fnids.append(fnid)
457         for target in self.build_targets:
458             if fnid in self.build_targets[target]:
459                 self.build_targets[target].remove(fnid)
460                 if len(self.build_targets[target]) == 0:
461                     self.remove_buildtarget(target)
462         for target in self.run_targets:
463             if fnid in self.run_targets[target]:
464                 self.run_targets[target].remove(fnid)
465                 if len(self.run_targets[target]) == 0:
466                     self.remove_runtarget(target)
467
468     def remove_buildtarget(self, targetid):
469         """
470         Mark a build target as failed (unbuildable)
471         Trigger removal of any files that have this as a dependency
472         """
473         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
474         self.failed_deps.append(targetid)
475         dependees = self.get_dependees(targetid)
476         for fnid in dependees:
477             self.fail_fnid(fnid)
478
479     def remove_runtarget(self, targetid):
480         """
481         Mark a run target as failed (unbuildable)
482         Trigger removal of any files that have this as a dependency
483         """
484         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
485         self.failed_rdeps.append(targetid)
486         dependees = self.get_rdependees(targetid)
487         for fnid in dependees:
488             self.fail_fnid(fnid)
489
490     def add_unresolved(self, cfgData, dataCache):
491         """
492         Resolve all unresolved build and runtime targets
493         """
494         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
495         while 1:
496             added = 0
497             for target in self.get_unresolved_build_targets(dataCache):
498                 try:
499                     self.add_provider(cfgData, dataCache, target)
500                     added = added + 1
501                 except bb.providers.NoProvider:
502                     # FIXME - should look at configuration.abort here and raise if set
503                     self.remove_buildtarget(self.getbuild_id(target))
504             for target in self.get_unresolved_run_targets(dataCache):
505                 try:
506                     self.add_rprovider(cfgData, dataCache, target)
507                     added = added + 1
508                 except bb.providers.NoRProvider:
509                     # FIXME - should look at configuration.abort here and raise if set
510                     self.remove_runtarget(self.getrun_id(target))
511             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
512             if added == 0:
513                 break
514             
515
516
517     def dump_data(self):
518         """
519         Dump some debug information on the internal data structures
520         """
521         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
522         bb.msg.debug(3, bb.msg.domain.TaskData, self.build_names_index)
523         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
524         bb.msg.debug(3, bb.msg.domain.TaskData, self.run_names_index)
525         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
526         for target in self.build_targets.keys():
527             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
528         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
529         for target in self.run_targets.keys():
530             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
531         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
532         for task in range(len(self.tasks_name)):
533             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
534                 task, 
535                 self.fn_index[self.tasks_fnid[task]], 
536                 self.tasks_name[task], 
537                 self.tasks_tdepends[task]))
538
539