taskqueue.py: Add tasks to be queried but not created
[bitbake.git] / lib / bb / taskdata.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 """
5 BitBake 'TaskData' implementation
6
7 Task data collection and handling
8
9 """
10
11 # Copyright (C) 2006  Richard Purdie
12 #
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License version 2 as
15 # published by the Free Software Foundation.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License along
23 # with this program; if not, write to the Free Software Foundation, Inc.,
24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25
26 from bb import data, fetch, event, mkdirhier, utils
27 import bb, os
28
29 class TaskData:
30     """
31     BitBake Task Data implementation
32     """
33     def __init__(self, abort = True):
34         self.build_names_index = []
35         self.run_names_index = []
36         self.fn_index = []
37
38         self.build_targets = {}
39         self.run_targets = {}
40
41         self.external_targets = []
42
43         self.tasks_fnid = []
44         self.tasks_name = []
45         self.tasks_tdepends = []
46         # Cache to speed up task ID lookups
47         self.tasks_lookup = {}
48
49         self.depids = {}
50         self.rdepids = {}
51
52         self.consider_msgs_cache = []
53
54         self.failed_deps = []
55         self.failed_rdeps = []
56         self.failed_fnids = []
57
58         self.abort = abort
59
60     def getbuild_id(self, name):
61         """
62         Return an ID number for the build target name.
63         If it doesn't exist, create one.
64         """
65         if not name in self.build_names_index:
66             self.build_names_index.append(name)
67             return len(self.build_names_index) - 1
68
69         return self.build_names_index.index(name)
70
71     def getrun_id(self, name):
72         """
73         Return an ID number for the run target name. 
74         If it doesn't exist, create one.
75         """
76         if not name in self.run_names_index:
77             self.run_names_index.append(name)
78             return len(self.run_names_index) - 1
79
80         return self.run_names_index.index(name)
81
82     def getfn_id(self, name):
83         """
84         Return an ID number for the filename. 
85         If it doesn't exist, create one.
86         """
87         if not name in self.fn_index:
88             self.fn_index.append(name)
89             return len(self.fn_index) - 1
90
91         return self.fn_index.index(name)
92
93     def gettask_id(self, fn, task, create = True):
94         """
95         Return an ID number for the task matching fn and task.
96         If it doesn't exist, create one by default.
97         Optionally return None instead.
98         """
99         fnid = self.getfn_id(fn)
100
101         if fnid in self.tasks_lookup:
102             if task in self.tasks_lookup[fnid]:
103                 return self.tasks_lookup[fnid][task]
104
105         if not create:
106             return None
107
108         self.tasks_name.append(task)
109         self.tasks_fnid.append(fnid)
110         self.tasks_tdepends.append([])
111
112         listid = len(self.tasks_name) - 1
113
114         if fnid not in self.tasks_lookup:
115             self.tasks_lookup[fnid] = {}
116         self.tasks_lookup[fnid][task] = listid
117
118         return listid
119
120     def add_tasks(self, fn, dataCache):
121         """
122         Add tasks for a given fn to the database
123         """
124
125         task_graph = dataCache.task_queues[fn]
126         task_deps = dataCache.task_deps[fn]
127
128         fnid = self.getfn_id(fn)
129
130         if fnid in self.failed_fnids:
131             bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
132
133         # Check if we've already seen this fn
134         if fnid in self.tasks_fnid:
135             return
136
137         # Work out task dependencies
138         for task in task_graph.allnodes():
139             parentids = []
140             for dep in task_graph.getparents(task):
141                 parentid = self.gettask_id(fn, dep)
142                 parentids.append(parentid)
143             taskid = self.gettask_id(fn, task)
144             self.tasks_tdepends[taskid].extend(parentids)
145
146         # Work out build dependencies
147         if not fnid in self.depids:
148             dependids = {}
149             for depend in dataCache.deps[fn]:
150                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
151                 dependids[self.getbuild_id(depend)] = None
152             self.depids[fnid] = dependids.keys()
153
154         # Work out runtime dependencies
155         if not fnid in self.rdepids:
156             rdependids = {}
157             rdepends = dataCache.rundeps[fn]
158             rrecs = dataCache.runrecs[fn]
159             for package in rdepends:
160                 for rdepend in rdepends[package]:
161                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
162                     rdependids[self.getrun_id(rdepend)] = None
163             for package in rrecs:
164                 for rdepend in rrecs[package]:
165                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
166                     rdependids[self.getrun_id(rdepend)] = None
167             self.rdepids[fnid] = rdependids.keys()
168
169         for dep in self.depids[fnid]:
170             if dep in self.failed_deps:
171                 self.fail_fnid(fnid)
172                 return
173         for dep in self.rdepids[fnid]:
174             if dep in self.failed_rdeps:
175                 self.fail_fnid(fnid)
176                 return
177
178     def have_build_target(self, target):
179         """
180         Have we a build target matching this name?
181         """
182         targetid = self.getbuild_id(target)
183
184         if targetid in self.build_targets:
185             return True
186         return False
187
188     def have_runtime_target(self, target):
189         """
190         Have we a runtime target matching this name?
191         """
192         targetid = self.getrun_id(target)
193
194         if targetid in self.run_targets:
195             return True
196         return False
197
198     def add_build_target(self, fn, item):
199         """
200         Add a build target.
201         If already present, append the provider fn to the list
202         """
203         targetid = self.getbuild_id(item)
204         fnid = self.getfn_id(fn)
205
206         if targetid in self.build_targets:
207             if fnid in self.build_targets[targetid]:
208                 return
209             self.build_targets[targetid].append(fnid)
210             return
211         self.build_targets[targetid] = [fnid]
212
213     def add_runtime_target(self, fn, item):
214         """
215         Add a runtime target.
216         If already present, append the provider fn to the list
217         """
218         targetid = self.getrun_id(item)
219         fnid = self.getfn_id(fn)
220
221         if targetid in self.run_targets:
222             if fnid in self.run_targets[targetid]:
223                 return
224             self.run_targets[targetid].append(fnid)
225             return
226         self.run_targets[targetid] = [fnid]
227
228     def mark_external_target(self, item):
229         """
230         Mark a build target as being externally requested
231         """
232         targetid = self.getbuild_id(item)
233
234         if targetid not in self.external_targets:
235             self.external_targets.append(targetid)
236
237     def get_unresolved_build_targets(self, dataCache):
238         """
239         Return a list of build targets who's providers 
240         are unknown.
241         """
242         unresolved = []
243         for target in self.build_names_index:
244             if target in dataCache.ignored_dependencies:
245                 continue
246             if self.build_names_index.index(target) in self.failed_deps:
247                 continue
248             if not self.have_build_target(target):
249                 unresolved.append(target)
250         return unresolved
251
252     def get_unresolved_run_targets(self, dataCache):
253         """
254         Return a list of runtime targets who's providers 
255         are unknown.
256         """
257         unresolved = []
258         for target in self.run_names_index:
259             if target in dataCache.ignored_dependencies:
260                 continue
261             if self.run_names_index.index(target) in self.failed_rdeps:
262                 continue
263             if not self.have_runtime_target(target):
264                 unresolved.append(target)
265         return unresolved
266
267     def get_provider(self, item):
268         """
269         Return a list of providers of item
270         """
271         targetid = self.getbuild_id(item)
272    
273         return self.build_targets[targetid]
274
275     def get_dependees(self, itemid):
276         """
277         Return a list of targets which depend on item
278         """
279         dependees = []
280         for fnid in self.depids:
281             if itemid in self.depids[fnid]:
282                 dependees.append(fnid)
283         return dependees
284
285     def get_dependees_str(self, item):
286         """
287         Return a list of targets which depend on item as a user readable string
288         """
289         itemid = self.getbuild_id(item)
290         dependees = []
291         for fnid in self.depids:
292             if itemid in self.depids[fnid]:
293                 dependees.append(self.fn_index[fnid])
294         return dependees
295
296     def get_rdependees(self, itemid):
297         """
298         Return a list of targets which depend on runtime item
299         """
300         dependees = []
301         for fnid in self.rdepids:
302             if itemid in self.rdepids[fnid]:
303                 dependees.append(fnid)
304         return dependees
305
306     def get_rdependees_str(self, item):
307         """
308         Return a list of targets which depend on runtime item as a user readable string
309         """
310         itemid = self.getrun_id(item)
311         dependees = []
312         for fnid in self.rdepids:
313             if itemid in self.rdepids[fnid]:
314                 dependees.append(self.fn_index[fnid])
315         return dependees
316
317     def add_provider(self, cfgData, dataCache, item):
318         try:
319             self.add_provider_internal(cfgData, dataCache, item)
320         except bb.providers.NoProvider:
321             if self.abort:
322                 bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
323                 raise
324             targetid = self.getbuild_id(item)
325             self.remove_buildtarget(targetid)
326
327         self.mark_external_target(item)
328
329     def add_provider_internal(self, cfgData, dataCache, item):
330         """
331         Add the providers of item to the task data
332         Mark entries were specifically added externally as against dependencies 
333         added internally during dependency resolution
334         """
335
336         if item in dataCache.ignored_dependencies:
337             return
338
339         if not item in dataCache.providers:
340             bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
341             bb.event.fire(bb.event.NoProvider(item, cfgData))
342             raise bb.providers.NoProvider(item)
343
344         if self.have_build_target(item):
345             return
346
347         all_p = dataCache.providers[item]
348
349         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
350
351         for p in eligible:
352             fnid = self.getfn_id(p)
353             if fnid in self.failed_fnids:
354                 eligible.remove(p)
355
356         if not eligible:
357             bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
358             bb.event.fire(bb.event.NoProvider(item, cfgData))
359             raise bb.providers.NoProvider(item)
360
361         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
362         if prefervar:
363             dataCache.preferred[item] = prefervar
364
365         discriminated = False
366         if item in dataCache.preferred:
367             for p in eligible:
368                 pn = dataCache.pkg_fn[p]
369                 if dataCache.preferred[item] == pn:
370                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
371                     eligible.remove(p)
372                     eligible = [p] + eligible
373                     discriminated = True
374                     break
375
376         if len(eligible) > 1 and discriminated == False:
377             if item not in self.consider_msgs_cache:
378                 providers_list = []
379                 for fn in eligible:
380                     providers_list.append(dataCache.pkg_fn[fn])
381                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
382                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
383                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
384             self.consider_msgs_cache.append(item)
385
386         for fn in eligible:
387             fnid = self.getfn_id(fn)
388             if fnid in self.failed_fnids:
389                 continue
390             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
391             self.add_build_target(fn, item)
392             self.add_tasks(fn, dataCache)
393
394
395             #item = dataCache.pkg_fn[fn]
396
397     def add_rprovider(self, cfgData, dataCache, item):
398         """
399         Add the runtime providers of item to the task data
400         (takes item names from RDEPENDS/PACKAGES namespace)
401         """
402
403         if item in dataCache.ignored_dependencies:
404             return
405
406         if self.have_runtime_target(item):
407             return
408
409         all_p = bb.providers.getRuntimeProviders(dataCache, item)
410
411         if not all_p:
412             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
413             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
414             raise bb.providers.NoRProvider(item)
415
416         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
417
418         for p in eligible:
419             fnid = self.getfn_id(p)
420             if fnid in self.failed_fnids:
421                 eligible.remove(p)
422
423         if not eligible:
424             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
425             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
426             raise bb.providers.NoRProvider(item)
427
428         # Should use dataCache.preferred here?
429         preferred = []
430         for p in eligible:
431             pn = dataCache.pkg_fn[p]
432             provides = dataCache.pn_provides[pn]
433             for provide in provides:
434                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
435                 if prefervar == pn:
436                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
437                     eligible.remove(p)
438                     eligible = [p] + eligible
439                     preferred.append(p)
440
441         if len(eligible) > 1 and len(preferred) == 0:
442             if item not in self.consider_msgs_cache:
443                 providers_list = []
444                 for fn in eligible:
445                     providers_list.append(dataCache.pkg_fn[fn])
446                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
447                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
448                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
449             self.consider_msgs_cache.append(item)
450
451         if len(preferred) > 1:
452             if item not in self.consider_msgs_cache:
453                 providers_list = []
454                 for fn in preferred:
455                     providers_list.append(dataCache.pkg_fn[fn])
456                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
457                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
458                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
459             self.consider_msgs_cache.append(item)
460
461         # run through the list until we find one that we can build
462         for fn in eligible:
463             fnid = self.getfn_id(fn)
464             if fnid in self.failed_fnids:
465                 continue
466             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
467             self.add_runtime_target(fn, item)
468             self.add_tasks(fn, dataCache)
469
470     def fail_fnid(self, fnid):
471         """
472         Mark a file as failed (unbuildable)
473         Remove any references from build and runtime provider lists
474         """
475         if fnid in self.failed_fnids:
476             return
477         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
478         self.failed_fnids.append(fnid)
479         for target in self.build_targets:
480             if fnid in self.build_targets[target]:
481                 self.build_targets[target].remove(fnid)
482                 if len(self.build_targets[target]) == 0:
483                     self.remove_buildtarget(target)
484         for target in self.run_targets:
485             if fnid in self.run_targets[target]:
486                 self.run_targets[target].remove(fnid)
487                 if len(self.run_targets[target]) == 0:
488                     self.remove_runtarget(target)
489
490     def remove_buildtarget(self, targetid):
491         """
492         Mark a build target as failed (unbuildable)
493         Trigger removal of any files that have this as a dependency
494         """
495         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
496         self.failed_deps.append(targetid)
497         dependees = self.get_dependees(targetid)
498         for fnid in dependees:
499             self.fail_fnid(fnid)
500         if self.abort and targetid in self.external_targets:
501             bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s" % self.build_names_index[targetid])
502             raise bb.providers.NoProvider
503
504     def remove_runtarget(self, targetid):
505         """
506         Mark a run target as failed (unbuildable)
507         Trigger removal of any files that have this as a dependency
508         """
509         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
510         self.failed_rdeps.append(targetid)
511         dependees = self.get_rdependees(targetid)
512         for fnid in dependees:
513             self.fail_fnid(fnid)
514
515     def add_unresolved(self, cfgData, dataCache):
516         """
517         Resolve all unresolved build and runtime targets
518         """
519         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
520         while 1:
521             added = 0
522             for target in self.get_unresolved_build_targets(dataCache):
523                 try:
524                     self.add_provider_internal(cfgData, dataCache, target)
525                     added = added + 1
526                 except bb.providers.NoProvider:
527                     targetid = self.getbuild_id(target)
528                     if self.abort and targetid in self.external_targets:
529                         raise
530                     self.remove_buildtarget(targetid)
531             for target in self.get_unresolved_run_targets(dataCache):
532                 try:
533                     self.add_rprovider(cfgData, dataCache, target)
534                     added = added + 1
535                 except bb.providers.NoRProvider:
536                     self.remove_runtarget(self.getrun_id(target))
537             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
538             if added == 0:
539                 break
540         # self.dump_data()
541
542     def dump_data(self):
543         """
544         Dump some debug information on the internal data structures
545         """
546         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
547         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index))
548         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
549         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index))
550         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
551         for target in self.build_targets.keys():
552             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
553         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
554         for target in self.run_targets.keys():
555             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
556         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
557         for task in range(len(self.tasks_name)):
558             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
559                 task, 
560                 self.fn_index[self.tasks_fnid[task]], 
561                 self.tasks_name[task], 
562                 self.tasks_tdepends[task]))
563         bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):")
564         for fnid in self.rdepids:
565             bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))
566
567