taskdata.py: Clean up mesage handling for unbuildable targets - log at debug level...
[bitbake.git] / lib / bb / taskdata.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 """
5 BitBake 'TaskData' implementation
6
7 Task data collection and handling
8
9 Copyright (C) 2006  Richard Purdie
10
11 This program is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License version 2 as published by the Free 
13 Software Foundation
14
15 This program is distributed in the hope that it will be useful, but WITHOUT
16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License along with
20 """
21
22 from bb import data, fetch, event, mkdirhier, utils
23 import bb, os
24
25 class TaskData:
26     """
27     BitBake Task Data implementation
28     """
29     def __init__(self):
30         self.build_names_index = []
31         self.run_names_index = []
32         self.fn_index = []
33
34         self.build_targets = {}
35         self.run_targets = {}
36
37         self.external_targets = []
38
39         self.tasks_fnid = []
40         self.tasks_name = []
41         self.tasks_tdepends = []
42
43         self.depids = {}
44         self.rdepids = {}
45
46         self.consider_msgs_cache = []
47
48         self.failed_deps = []
49         self.failed_rdeps = []
50         self.failed_fnids = []
51
52
53     def matches_in_list(self, data, substring):
54         """
55         Return a list of the positions of substring in list data
56         """
57         matches = []
58         start = 0
59         while 1:
60             try:
61                 start = data.index(substring, start)
62             except ValueError:
63                 return matches
64             matches.append(start)
65             start = start + 1
66
67     def both_contain(self, list1, list2):
68         """
69         Return the items present in both list1 and list2
70         """
71         matches = []
72         for data in list1:
73             if data in list2:
74                 return data
75         return None
76
77
78     def getbuild_id(self, name):
79         """
80         Return an ID number for the build target name.
81         If it doesn't exist, create one.
82         """
83         if not name in self.build_names_index:
84             self.build_names_index.append(name)
85
86         return self.build_names_index.index(name)
87
88     def getrun_id(self, name):
89         """
90         Return an ID number for the run target name. 
91         If it doesn't exist, create one.
92         """
93         if not name in self.run_names_index:
94             self.run_names_index.append(name)
95
96         return self.run_names_index.index(name)
97
98     def getfn_id(self, name):
99         """
100         Return an ID number for the filename. 
101         If it doesn't exist, create one.
102         """
103         if not name in self.fn_index:
104             self.fn_index.append(name)
105
106         return self.fn_index.index(name)
107
108     def gettask_id(self, fn, task):
109         """
110         Return an ID number for the task matching fn and task.
111         If it doesn't exist, create one.
112         """
113         fnid = self.getfn_id(fn)
114
115         fnids = self.matches_in_list(self.tasks_fnid, fnid)
116         names = self.matches_in_list(self.tasks_name, task)
117
118         listid = self.both_contain(fnids, names)
119
120         if listid is not None:
121             return listid
122
123         self.tasks_name.append(task)
124         self.tasks_fnid.append(fnid)
125         self.tasks_tdepends.append([])
126
127         return len(self.tasks_name)-1
128
129     def add_tasks(self, fn, dataCache):
130         """
131         Add tasks for a given fn to the database
132         """
133
134         task_graph = dataCache.task_queues[fn]
135         task_deps = dataCache.task_deps[fn]
136
137         fnid = self.getfn_id(fn)
138
139         if fnid in self.failed_fnids:
140             bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
141
142         # Check if we've already seen this fn
143         if fnid in self.tasks_fnid:
144             return
145
146         # Work out task dependencies
147         for task in task_graph.allnodes():
148             parentids = []
149             for dep in task_graph.getparents(task):
150                 parentid = self.gettask_id(fn, dep)
151                 parentids.append(parentid)
152             taskid = self.gettask_id(fn, task)
153             self.tasks_tdepends[taskid].extend(parentids)
154
155         # Work out build dependencies
156         if not fnid in self.depids:
157             dependids = []
158             for depend in dataCache.deps[fn]:
159                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
160                 dependids.append(self.getbuild_id(depend))
161             self.depids[fnid] = dependids
162
163         # Work out runtime dependencies
164         if not fnid in self.rdepids:
165             rdependids = []
166             rdepends = dataCache.rundeps[fn]
167             rrecs = dataCache.runrecs[fn]
168             for package in rdepends:
169                 for rdepend in rdepends[package]:
170                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
171                     rdependids.append(self.getrun_id(rdepend))
172             for package in rrecs:
173                 for rdepend in rrecs[package]:
174                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
175                     rdependids.append(self.getrun_id(rdepend))
176             self.rdepids[fnid] = rdependids
177
178     def have_build_target(self, target):
179         """
180         Have we a build target matching this name?
181         """
182         targetid = self.getbuild_id(target)
183
184         if targetid in self.build_targets:
185             return True
186         return False
187
188     def have_runtime_target(self, target):
189         """
190         Have we a runtime target matching this name?
191         """
192         targetid = self.getrun_id(target)
193
194         if targetid in self.run_targets:
195             return True
196         return False
197
198     def add_build_target(self, fn, item):
199         """
200         Add a build target.
201         If already present, append the provider fn to the list
202         """
203         targetid = self.getbuild_id(item)
204         fnid = self.getfn_id(fn)
205
206         if targetid in self.build_targets:
207             if fnid in self.build_targets[targetid]:
208                 return
209             self.build_targets[targetid].append(fnid)
210             return
211         self.build_targets[targetid] = [fnid]
212
213     def add_runtime_target(self, fn, item):
214         """
215         Add a runtime target.
216         If already present, append the provider fn to the list
217         """
218         targetid = self.getrun_id(item)
219         fnid = self.getfn_id(fn)
220
221         if targetid in self.run_targets:
222             if fnid in self.run_targets[targetid]:
223                 return
224             self.run_targets[targetid].append(fnid)
225             return
226         self.run_targets[targetid] = [fnid]
227
228     def mark_external_target(self, item):
229         """
230         Mark a build target as being externally requested
231         """
232         targetid = self.getbuild_id(item)
233
234         if targetid not in self.external_targets:
235             self.external_targets.append(targetid)
236
237     def get_unresolved_build_targets(self, dataCache):
238         """
239         Return a list of build targets who's providers 
240         are unknown.
241         """
242         unresolved = []
243         for target in self.build_names_index:
244             if target in dataCache.ignored_dependencies:
245                 continue
246             if self.build_names_index.index(target) in self.failed_deps:
247                 continue
248             if not self.have_build_target(target):
249                 unresolved.append(target)
250         return unresolved
251
252     def get_unresolved_run_targets(self, dataCache):
253         """
254         Return a list of runtime targets who's providers 
255         are unknown.
256         """
257         unresolved = []
258         for target in self.run_names_index:
259             if target in dataCache.ignored_dependencies:
260                 continue
261             if self.run_names_index.index(target) in self.failed_rdeps:
262                 continue
263             if not self.have_runtime_target(target):
264                 unresolved.append(target)
265         return unresolved
266
267     def get_provider(self, item):
268         """
269         Return a list of providers of item
270         """
271         targetid = self.getbuild_id(item)
272    
273         return self.build_targets[targetid]
274
275     def get_dependees(self, itemid):
276         """
277         Return a list of targets which depend on item
278         """
279         dependees = []
280         for fnid in self.depids:
281             if itemid in self.depids[fnid]:
282                 dependees.append(fnid)
283         return dependees
284
285     def get_dependees_str(self, item):
286         """
287         Return a list of targets which depend on item as a user readable string
288         """
289         itemid = self.getbuild_id(item)
290         dependees = []
291         for fnid in self.depids:
292             if itemid in self.depids[fnid]:
293                 dependees.append(self.fn_index[fnid])
294         return dependees
295
296     def get_rdependees(self, itemid):
297         """
298         Return a list of targets which depend on runtime item
299         """
300         dependees = []
301         for fnid in self.rdepids:
302             if itemid in self.rdepids[fnid]:
303                 dependees.append(fnid)
304         return dependees
305
306     def get_rdependees_str(self, item):
307         """
308         Return a list of targets which depend on runtime item as a user readable string
309         """
310         itemid = self.getrun_id(item)
311         dependees = []
312         for fnid in self.rdepids:
313             if itemid in self.rdepids[fnid]:
314                 dependees.append(self.fn_index[fnid])
315         return dependees
316
317     def add_provider(self, cfgData, dataCache, item, external = True):
318         """
319         Add the providers of item to the task data
320         Mark entries were specifically added externally as against dependencies 
321         added internally during dependency resolution
322         """
323
324         if item in dataCache.ignored_dependencies:
325             return True
326
327         if not item in dataCache.providers:
328             msg = "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item))
329             if external:
330                 bb.msg.error(bb.msg.domain.Provider, msg)
331             else:
332                 bb.msg.debug(1, bb.msg.domain.Provider, msg)
333             bb.event.fire(bb.event.NoProvider(item, cfgData))
334             raise bb.providers.NoProvider(item)
335
336         if self.have_build_target(item):
337             return True
338
339         all_p = dataCache.providers[item]
340
341         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
342
343         for p in eligible:
344             fnid = self.getfn_id(p)
345             if fnid in self.failed_fnids:
346                 eligible.remove(p)
347
348         if not eligible:
349             msg = "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item))
350             if external:
351                 bb.msg.error(bb.msg.domain.Provider, msg)
352             else:
353                 bb.msg.debug(1, bb.msg.domain.Provider, msg)
354             bb.event.fire(bb.event.NoProvider(item, cfgData))
355             raise bb.providers.NoProvider(item)
356
357         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
358         if prefervar:
359             dataCache.preferred[item] = prefervar
360
361         discriminated = False
362         if item in dataCache.preferred:
363             for p in eligible:
364                 pn = dataCache.pkg_fn[p]
365                 if dataCache.preferred[item] == pn:
366                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
367                     eligible.remove(p)
368                     eligible = [p] + eligible
369                     discriminated = True
370                     break
371
372         if len(eligible) > 1 and discriminated == False:
373             if item not in self.consider_msgs_cache:
374                 providers_list = []
375                 for fn in eligible:
376                     providers_list.append(dataCache.pkg_fn[fn])
377                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
378                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
379                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
380             self.consider_msgs_cache.append(item)
381
382         for fn in eligible:
383             fnid = self.getfn_id(fn)
384             if fnid in self.failed_fnids:
385                 continue
386             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
387             self.add_tasks(fn, dataCache)
388             self.add_build_target(fn, item)
389
390             item = dataCache.pkg_fn[fn]
391
392         self.mark_external_target(item)
393
394         return True
395
396     def add_rprovider(self, cfgData, dataCache, item):
397         """
398         Add the runtime providers of item to the task data
399         (takes item names from RDEPENDS/PACKAGES namespace)
400         """
401
402         if item in dataCache.ignored_dependencies:
403             return True
404
405         if self.have_runtime_target(item):
406             return True
407
408         all_p = bb.providers.getRuntimeProviders(dataCache, item)
409
410         if not all_p:
411             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
412             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
413             raise bb.providers.NoRProvider(item)
414
415         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
416
417         for p in eligible:
418             fnid = self.getfn_id(p)
419             if fnid in self.failed_fnids:
420                 eligible.remove(p)
421
422         if not eligible:
423             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
424             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
425             raise bb.providers.NoRProvider(item)
426
427         # Should use dataCache.preferred here?
428         preferred = []
429         for p in eligible:
430             pn = dataCache.pkg_fn[p]
431             provides = dataCache.pn_provides[pn]
432             for provide in provides:
433                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
434                 if prefervar == pn:
435                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
436                     eligible.remove(p)
437                     eligible = [p] + eligible
438                     preferred.append(p)
439
440         if len(eligible) > 1 and len(preferred) == 0:
441             if item not in self.consider_msgs_cache:
442                 providers_list = []
443                 for fn in eligible:
444                     providers_list.append(dataCache.pkg_fn[fn])
445                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
446                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
447                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
448             self.consider_msgs_cache.append(item)
449
450         if len(preferred) > 1:
451             if item not in self.consider_msgs_cache:
452                 providers_list = []
453                 for fn in preferred:
454                     providers_list.append(dataCache.pkg_fn[fn])
455                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available (%s);" % ", ".join(providers_list))
456                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
457                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
458             self.consider_msgs_cache.append(item)
459
460         # run through the list until we find one that we can build
461         for fn in eligible:
462             fnid = self.getfn_id(fn)
463             if fnid in self.failed_fnids:
464                 continue
465             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
466             self.add_tasks(fn, dataCache)
467             self.add_runtime_target(fn, item)
468
469         return True
470
471     def fail_fnid(self, fnid):
472         """
473         Mark a file as failed (unbuildable)
474         Remove any references from build and runtime provider lists
475         """
476         if fnid in self.failed_fnids:
477             return
478         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
479         self.failed_fnids.append(fnid)
480         for target in self.build_targets:
481             if fnid in self.build_targets[target]:
482                 self.build_targets[target].remove(fnid)
483                 if len(self.build_targets[target]) == 0:
484                     self.remove_buildtarget(target)
485         for target in self.run_targets:
486             if fnid in self.run_targets[target]:
487                 self.run_targets[target].remove(fnid)
488                 if len(self.run_targets[target]) == 0:
489                     self.remove_runtarget(target)
490
491     def remove_buildtarget(self, targetid):
492         """
493         Mark a build target as failed (unbuildable)
494         Trigger removal of any files that have this as a dependency
495         """
496         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
497         self.failed_deps.append(targetid)
498         dependees = self.get_dependees(targetid)
499         for fnid in dependees:
500             self.fail_fnid(fnid)
501
502     def remove_runtarget(self, targetid):
503         """
504         Mark a run target as failed (unbuildable)
505         Trigger removal of any files that have this as a dependency
506         """
507         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
508         self.failed_rdeps.append(targetid)
509         dependees = self.get_rdependees(targetid)
510         for fnid in dependees:
511             self.fail_fnid(fnid)
512
513     def add_unresolved(self, cfgData, dataCache):
514         """
515         Resolve all unresolved build and runtime targets
516         """
517         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
518         while 1:
519             added = 0
520             for target in self.get_unresolved_build_targets(dataCache):
521                 try:
522                     self.add_provider(cfgData, dataCache, target, False)
523                     added = added + 1
524                 except bb.providers.NoProvider:
525                     targetid = self.getbuild_id(target)
526                     if targetid in self.external_targets:
527                         # FIXME - should look at configuration.abort here and only raise if set
528                         raise
529                     self.remove_buildtarget(targetid)
530             for target in self.get_unresolved_run_targets(dataCache):
531                 try:
532                     self.add_rprovider(cfgData, dataCache, target)
533                     added = added + 1
534                 except bb.providers.NoRProvider:
535                     self.remove_runtarget(self.getrun_id(target))
536             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
537             if added == 0:
538                 break
539
540
541     def dump_data(self):
542         """
543         Dump some debug information on the internal data structures
544         """
545         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
546         bb.msg.debug(3, bb.msg.domain.TaskData, self.build_names_index)
547         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
548         bb.msg.debug(3, bb.msg.domain.TaskData, self.run_names_index)
549         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
550         for target in self.build_targets.keys():
551             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
552         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
553         for target in self.run_targets.keys():
554             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
555         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
556         for task in range(len(self.tasks_name)):
557             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
558                 task, 
559                 self.fn_index[self.tasks_fnid[task]], 
560                 self.tasks_name[task], 
561                 self.tasks_tdepends[task]))
562
563