SignatureGenerator: Add empty implementation for dump_sigs
[bitbake.git] / lib / bb / taskdata.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 """
5 BitBake 'TaskData' implementation
6
7 Task data collection and handling
8
9 """
10
11 # Copyright (C) 2006  Richard Purdie
12 #
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License version 2 as
15 # published by the Free Software Foundation.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License along
23 # with this program; if not, write to the Free Software Foundation, Inc.,
24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25
26 import logging
27 import re
28 import bb
29
30 logger = logging.getLogger("BitBake.TaskData")
31
32 def re_match_strings(target, strings):
33     """
34     Whether or not the string 'target' matches
35     any one string of the strings which can be regular expression string
36     """
37     return any(name == target or re.match(name, target)
38                for name in strings)
39
40 class TaskData:
41     """
42     BitBake Task Data implementation
43     """
44     def __init__(self, abort = True, tryaltconfigs = False, skiplist = None):
45         self.build_names_index = []
46         self.run_names_index = []
47         self.fn_index = []
48
49         self.build_targets = {}
50         self.run_targets = {}
51
52         self.external_targets = []
53
54         self.tasks_fnid = []
55         self.tasks_name = []
56         self.tasks_tdepends = []
57         self.tasks_idepends = []
58         self.tasks_irdepends = []
59         # Cache to speed up task ID lookups
60         self.tasks_lookup = {}
61
62         self.depids = {}
63         self.rdepids = {}
64
65         self.consider_msgs_cache = []
66
67         self.failed_deps = []
68         self.failed_rdeps = []
69         self.failed_fnids = []
70
71         self.abort = abort
72         self.tryaltconfigs = tryaltconfigs
73
74         self.skiplist = skiplist
75
76     def getbuild_id(self, name):
77         """
78         Return an ID number for the build target name.
79         If it doesn't exist, create one.
80         """
81         if not name in self.build_names_index:
82             self.build_names_index.append(name)
83             return len(self.build_names_index) - 1
84
85         return self.build_names_index.index(name)
86
87     def getrun_id(self, name):
88         """
89         Return an ID number for the run target name.
90         If it doesn't exist, create one.
91         """
92         if not name in self.run_names_index:
93             self.run_names_index.append(name)
94             return len(self.run_names_index) - 1
95
96         return self.run_names_index.index(name)
97
98     def getfn_id(self, name):
99         """
100         Return an ID number for the filename.
101         If it doesn't exist, create one.
102         """
103         if not name in self.fn_index:
104             self.fn_index.append(name)
105             return len(self.fn_index) - 1
106
107         return self.fn_index.index(name)
108
109     def gettask_ids(self, fnid):
110         """
111         Return an array of the ID numbers matching a given fnid.
112         """
113         ids = []
114         if fnid in self.tasks_lookup:
115             for task in self.tasks_lookup[fnid]:
116                 ids.append(self.tasks_lookup[fnid][task])
117         return ids
118
119     def gettask_id_fromfnid(self, fnid, task):
120         """
121         Return an ID number for the task matching fnid and task.
122         """
123         if fnid in self.tasks_lookup:
124             if task in self.tasks_lookup[fnid]:
125                 return self.tasks_lookup[fnid][task]
126
127         return None
128
129     def gettask_id(self, fn, task, create = True):
130         """
131         Return an ID number for the task matching fn and task.
132         If it doesn't exist, create one by default.
133         Optionally return None instead.
134         """
135         fnid = self.getfn_id(fn)
136
137         if fnid in self.tasks_lookup:
138             if task in self.tasks_lookup[fnid]:
139                 return self.tasks_lookup[fnid][task]
140
141         if not create:
142             return None
143
144         self.tasks_name.append(task)
145         self.tasks_fnid.append(fnid)
146         self.tasks_tdepends.append([])
147         self.tasks_idepends.append([])
148         self.tasks_irdepends.append([])
149
150         listid = len(self.tasks_name) - 1
151
152         if fnid not in self.tasks_lookup:
153             self.tasks_lookup[fnid] = {}
154         self.tasks_lookup[fnid][task] = listid
155
156         return listid
157
158     def add_tasks(self, fn, dataCache):
159         """
160         Add tasks for a given fn to the database
161         """
162
163         task_deps = dataCache.task_deps[fn]
164
165         fnid = self.getfn_id(fn)
166
167         if fnid in self.failed_fnids:
168             bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
169
170         # Check if we've already seen this fn
171         if fnid in self.tasks_fnid:
172             return
173
174         for task in task_deps['tasks']:
175
176             # Work out task dependencies
177             parentids = []
178             for dep in task_deps['parents'][task]:
179                 if dep not in task_deps['tasks']:
180                     bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
181                     continue
182                 parentid = self.gettask_id(fn, dep)
183                 parentids.append(parentid)
184             taskid = self.gettask_id(fn, task)
185             self.tasks_tdepends[taskid].extend(parentids)
186
187             # Touch all intertask dependencies
188             if 'depends' in task_deps and task in task_deps['depends']:
189                 ids = []
190                 for dep in task_deps['depends'][task].split():
191                     if dep:
192                         if ":" not in dep:
193                             bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
194                         ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
195                 self.tasks_idepends[taskid].extend(ids)
196             if 'rdepends' in task_deps and task in task_deps['rdepends']:
197                 ids = []
198                 for dep in task_deps['rdepends'][task].split():
199                     if dep:
200                         if ":" not in dep:
201                             bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
202                         ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1]))
203                 self.tasks_irdepends[taskid].extend(ids)
204
205
206         # Work out build dependencies
207         if not fnid in self.depids:
208             dependids = {}
209             for depend in dataCache.deps[fn]:
210                 logger.debug(2, "Added dependency %s for %s", depend, fn)
211                 dependids[self.getbuild_id(depend)] = None
212             self.depids[fnid] = dependids.keys()
213
214         # Work out runtime dependencies
215         if not fnid in self.rdepids:
216             rdependids = {}
217             rdepends = dataCache.rundeps[fn]
218             rrecs = dataCache.runrecs[fn]
219             for package in rdepends:
220                 for rdepend in rdepends[package]:
221                     logger.debug(2, "Added runtime dependency %s for %s", rdepend, fn)
222                     rdependids[self.getrun_id(rdepend)] = None
223             for package in rrecs:
224                 for rdepend in rrecs[package]:
225                     logger.debug(2, "Added runtime recommendation %s for %s", rdepend, fn)
226                     rdependids[self.getrun_id(rdepend)] = None
227             self.rdepids[fnid] = rdependids.keys()
228
229         for dep in self.depids[fnid]:
230             if dep in self.failed_deps:
231                 self.fail_fnid(fnid)
232                 return
233         for dep in self.rdepids[fnid]:
234             if dep in self.failed_rdeps:
235                 self.fail_fnid(fnid)
236                 return
237
238     def have_build_target(self, target):
239         """
240         Have we a build target matching this name?
241         """
242         targetid = self.getbuild_id(target)
243
244         if targetid in self.build_targets:
245             return True
246         return False
247
248     def have_runtime_target(self, target):
249         """
250         Have we a runtime target matching this name?
251         """
252         targetid = self.getrun_id(target)
253
254         if targetid in self.run_targets:
255             return True
256         return False
257
258     def add_build_target(self, fn, item):
259         """
260         Add a build target.
261         If already present, append the provider fn to the list
262         """
263         targetid = self.getbuild_id(item)
264         fnid = self.getfn_id(fn)
265
266         if targetid in self.build_targets:
267             if fnid in self.build_targets[targetid]:
268                 return
269             self.build_targets[targetid].append(fnid)
270             return
271         self.build_targets[targetid] = [fnid]
272
273     def add_runtime_target(self, fn, item):
274         """
275         Add a runtime target.
276         If already present, append the provider fn to the list
277         """
278         targetid = self.getrun_id(item)
279         fnid = self.getfn_id(fn)
280
281         if targetid in self.run_targets:
282             if fnid in self.run_targets[targetid]:
283                 return
284             self.run_targets[targetid].append(fnid)
285             return
286         self.run_targets[targetid] = [fnid]
287
288     def mark_external_target(self, item):
289         """
290         Mark a build target as being externally requested
291         """
292         targetid = self.getbuild_id(item)
293
294         if targetid not in self.external_targets:
295             self.external_targets.append(targetid)
296
297     def get_unresolved_build_targets(self, dataCache):
298         """
299         Return a list of build targets who's providers
300         are unknown.
301         """
302         unresolved = []
303         for target in self.build_names_index:
304             if re_match_strings(target, dataCache.ignored_dependencies):
305                 continue
306             if self.build_names_index.index(target) in self.failed_deps:
307                 continue
308             if not self.have_build_target(target):
309                 unresolved.append(target)
310         return unresolved
311
312     def get_unresolved_run_targets(self, dataCache):
313         """
314         Return a list of runtime targets who's providers
315         are unknown.
316         """
317         unresolved = []
318         for target in self.run_names_index:
319             if re_match_strings(target, dataCache.ignored_dependencies):
320                 continue
321             if self.run_names_index.index(target) in self.failed_rdeps:
322                 continue
323             if not self.have_runtime_target(target):
324                 unresolved.append(target)
325         return unresolved
326
327     def get_provider(self, item):
328         """
329         Return a list of providers of item
330         """
331         targetid = self.getbuild_id(item)
332
333         return self.build_targets[targetid]
334
335     def get_dependees(self, itemid):
336         """
337         Return a list of targets which depend on item
338         """
339         dependees = []
340         for fnid in self.depids:
341             if itemid in self.depids[fnid]:
342                 dependees.append(fnid)
343         return dependees
344
345     def get_dependees_str(self, item):
346         """
347         Return a list of targets which depend on item as a user readable string
348         """
349         itemid = self.getbuild_id(item)
350         dependees = []
351         for fnid in self.depids:
352             if itemid in self.depids[fnid]:
353                 dependees.append(self.fn_index[fnid])
354         return dependees
355
356     def get_rdependees(self, itemid):
357         """
358         Return a list of targets which depend on runtime item
359         """
360         dependees = []
361         for fnid in self.rdepids:
362             if itemid in self.rdepids[fnid]:
363                 dependees.append(fnid)
364         return dependees
365
366     def get_rdependees_str(self, item):
367         """
368         Return a list of targets which depend on runtime item as a user readable string
369         """
370         itemid = self.getrun_id(item)
371         dependees = []
372         for fnid in self.rdepids:
373             if itemid in self.rdepids[fnid]:
374                 dependees.append(self.fn_index[fnid])
375         return dependees
376
377     def get_reasons(self, item, runtime=False):
378         """
379         Get the reason(s) for an item not being provided, if any
380         """
381         reasons = []
382         if self.skiplist:
383             for fn in self.skiplist:
384                 skipitem = self.skiplist[fn]
385                 if skipitem.pn == item:
386                     reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
387                 elif runtime and item in skipitem.rprovides:
388                     reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
389                 elif not runtime and item in skipitem.provides:
390                     reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
391         return reasons
392
393     def get_close_matches(self, item, provider_list):
394         import difflib
395         if self.skiplist:
396             skipped = []
397             for fn in self.skiplist:
398                 skipped.append(self.skiplist[fn].pn)
399             full_list = provider_list + skipped
400         else:
401             full_list = provider_list
402         return difflib.get_close_matches(item, full_list, cutoff=0.7)
403
404     def add_provider(self, cfgData, dataCache, item):
405         try:
406             self.add_provider_internal(cfgData, dataCache, item)
407         except bb.providers.NoProvider:
408             if self.abort:
409                 raise
410             self.remove_buildtarget(self.getbuild_id(item))
411
412         self.mark_external_target(item)
413
414     def add_provider_internal(self, cfgData, dataCache, item):
415         """
416         Add the providers of item to the task data
417         Mark entries were specifically added externally as against dependencies
418         added internally during dependency resolution
419         """
420
421         if re_match_strings(item, dataCache.ignored_dependencies):
422             return
423
424         if not item in dataCache.providers:
425             bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=self.get_close_matches(item, dataCache.providers.keys())), cfgData)
426             raise bb.providers.NoProvider(item)
427
428         if self.have_build_target(item):
429             return
430
431         all_p = dataCache.providers[item]
432
433         eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
434         eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
435
436         if not eligible:
437             bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
438             raise bb.providers.NoProvider(item)
439
440         if len(eligible) > 1 and foundUnique == False:
441             if item not in self.consider_msgs_cache:
442                 providers_list = []
443                 for fn in eligible:
444                     providers_list.append(dataCache.pkg_fn[fn])
445                 bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
446             self.consider_msgs_cache.append(item)
447
448         for fn in eligible:
449             fnid = self.getfn_id(fn)
450             if fnid in self.failed_fnids:
451                 continue
452             logger.debug(2, "adding %s to satisfy %s", fn, item)
453             self.add_build_target(fn, item)
454             self.add_tasks(fn, dataCache)
455
456
457             #item = dataCache.pkg_fn[fn]
458
459     def add_rprovider(self, cfgData, dataCache, item):
460         """
461         Add the runtime providers of item to the task data
462         (takes item names from RDEPENDS/PACKAGES namespace)
463         """
464
465         if re_match_strings(item, dataCache.ignored_dependencies):
466             return
467
468         if self.have_runtime_target(item):
469             return
470
471         all_p = bb.providers.getRuntimeProviders(dataCache, item)
472
473         if not all_p:
474             bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData)
475             raise bb.providers.NoRProvider(item)
476
477         eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
478         eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
479
480         if not eligible:
481             bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
482             raise bb.providers.NoRProvider(item)
483
484         if len(eligible) > 1 and numberPreferred == 0:
485             if item not in self.consider_msgs_cache:
486                 providers_list = []
487                 for fn in eligible:
488                     providers_list.append(dataCache.pkg_fn[fn])
489                 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
490             self.consider_msgs_cache.append(item)
491
492         if numberPreferred > 1:
493             if item not in self.consider_msgs_cache:
494                 providers_list = []
495                 for fn in eligible:
496                     providers_list.append(dataCache.pkg_fn[fn])
497                 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
498             self.consider_msgs_cache.append(item)
499             raise bb.providers.MultipleRProvider(item)
500
501         # run through the list until we find one that we can build
502         for fn in eligible:
503             fnid = self.getfn_id(fn)
504             if fnid in self.failed_fnids:
505                 continue
506             logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
507             self.add_runtime_target(fn, item)
508             self.add_tasks(fn, dataCache)
509
510     def fail_fnid(self, fnid, missing_list = []):
511         """
512         Mark a file as failed (unbuildable)
513         Remove any references from build and runtime provider lists
514
515         missing_list, A list of missing requirements for this target
516         """
517         if fnid in self.failed_fnids:
518             return
519         logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
520         self.failed_fnids.append(fnid)
521         for target in self.build_targets:
522             if fnid in self.build_targets[target]:
523                 self.build_targets[target].remove(fnid)
524                 if len(self.build_targets[target]) == 0:
525                     self.remove_buildtarget(target, missing_list)
526         for target in self.run_targets:
527             if fnid in self.run_targets[target]:
528                 self.run_targets[target].remove(fnid)
529                 if len(self.run_targets[target]) == 0:
530                     self.remove_runtarget(target, missing_list)
531
532     def remove_buildtarget(self, targetid, missing_list = []):
533         """
534         Mark a build target as failed (unbuildable)
535         Trigger removal of any files that have this as a dependency
536         """
537         if not missing_list:
538             missing_list = [self.build_names_index[targetid]]
539         else:
540             missing_list = [self.build_names_index[targetid]] + missing_list
541         logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list)
542         self.failed_deps.append(targetid)
543         dependees = self.get_dependees(targetid)
544         for fnid in dependees:
545             self.fail_fnid(fnid, missing_list)
546         for taskid in xrange(len(self.tasks_idepends)):
547             idepends = self.tasks_idepends[taskid]
548             for (idependid, idependtask) in idepends:
549                 if idependid == targetid:
550                     self.fail_fnid(self.tasks_fnid[taskid], missing_list)
551
552         if self.abort and targetid in self.external_targets:
553             target = self.build_names_index[targetid]
554             logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
555             raise bb.providers.NoProvider(target)
556
557     def remove_runtarget(self, targetid, missing_list = []):
558         """
559         Mark a run target as failed (unbuildable)
560         Trigger removal of any files that have this as a dependency
561         """
562         if not missing_list:
563             missing_list = [self.run_names_index[targetid]]
564         else:
565             missing_list = [self.run_names_index[targetid]] + missing_list
566
567         logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list)
568         self.failed_rdeps.append(targetid)
569         dependees = self.get_rdependees(targetid)
570         for fnid in dependees:
571             self.fail_fnid(fnid, missing_list)
572         for taskid in xrange(len(self.tasks_irdepends)):
573             irdepends = self.tasks_irdepends[taskid]
574             for (idependid, idependtask) in irdepends:
575                 if idependid == targetid:
576                     self.fail_fnid(self.tasks_fnid[taskid], missing_list)
577
578     def add_unresolved(self, cfgData, dataCache):
579         """
580         Resolve all unresolved build and runtime targets
581         """
582         logger.info("Resolving any missing task queue dependencies")
583         while True:
584             added = 0
585             for target in self.get_unresolved_build_targets(dataCache):
586                 try:
587                     self.add_provider_internal(cfgData, dataCache, target)
588                     added = added + 1
589                 except bb.providers.NoProvider:
590                     targetid = self.getbuild_id(target)
591                     if self.abort and targetid in self.external_targets:
592                         raise
593                     self.remove_buildtarget(targetid)
594             for target in self.get_unresolved_run_targets(dataCache):
595                 try:
596                     self.add_rprovider(cfgData, dataCache, target)
597                     added = added + 1
598                 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
599                     self.remove_runtarget(self.getrun_id(target))
600             logger.debug(1, "Resolved " + str(added) + " extra dependencies")
601             if added == 0:
602                 break
603         # self.dump_data()
604
605     def dump_data(self):
606         """
607         Dump some debug information on the internal data structures
608         """
609         logger.debug(3, "build_names:")
610         logger.debug(3, ", ".join(self.build_names_index))
611
612         logger.debug(3, "run_names:")
613         logger.debug(3, ", ".join(self.run_names_index))
614
615         logger.debug(3, "build_targets:")
616         for buildid in xrange(len(self.build_names_index)):
617             target = self.build_names_index[buildid]
618             targets = "None"
619             if buildid in self.build_targets:
620                 targets = self.build_targets[buildid]
621             logger.debug(3, " (%s)%s: %s", buildid, target, targets)
622
623         logger.debug(3, "run_targets:")
624         for runid in xrange(len(self.run_names_index)):
625             target = self.run_names_index[runid]
626             targets = "None"
627             if runid in self.run_targets:
628                 targets = self.run_targets[runid]
629             logger.debug(3, " (%s)%s: %s", runid, target, targets)
630
631         logger.debug(3, "tasks:")
632         for task in xrange(len(self.tasks_name)):
633             logger.debug(3, " (%s)%s - %s: %s",
634                        task,
635                        self.fn_index[self.tasks_fnid[task]],
636                        self.tasks_name[task],
637                        self.tasks_tdepends[task])
638
639         logger.debug(3, "dependency ids (per fn):")
640         for fnid in self.depids:
641             logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid])
642
643         logger.debug(3, "runtime dependency ids (per fn):")
644         for fnid in self.rdepids:
645             logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid])