runqueue.py: Fix exit code for build failures in --continue mode
[bitbake.git] / lib / bb / runqueue.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 """
5 BitBake 'RunQueue' implementation
6
7 Handles preparation and execution of a queue of tasks
8 """
9
10 # Copyright (C) 2006-2007  Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 from bb import msg, data, event, mkdirhier, utils
26 from sets import Set 
27 import bb, os, sys
28 import signal
29
30 class TaskFailure(Exception):
31     """Exception raised when a task in a runqueue fails"""
32     def __init__(self, x): 
33         self.args = x
34
35
36 class RunQueueStats:
37     """
38     Holds statistics on the tasks handled by the associated runQueue
39     """
40     def __init__(self):
41         self.completed = 0
42         self.skipped = 0
43         self.failed = 0
44
45     def taskFailed(self):
46         self.failed = self.failed + 1
47
48     def taskCompleted(self):
49         self.completed = self.completed + 1
50
51     def taskSkipped(self):
52         self.skipped = self.skipped + 1
53
54 class RunQueueScheduler:
55     """
56     Control the order tasks are scheduled in.
57     """
58     def __init__(self, runqueue):
59         """
60         The default scheduler just returns the first buildable task (the 
61         priority map is sorted by task numer)
62         """
63         self.rq = runqueue
64         numTasks = len(self.rq.runq_fnid)
65
66         self.prio_map = []
67         self.prio_map.extend(range(numTasks))
68
69     def next(self):
70         """
71         Return the id of the first task we find that is buildable
72         """
73         for task1 in range(len(self.rq.runq_fnid)):
74             task = self.prio_map[task1]
75             if self.rq.runq_running[task] == 1:
76                 continue
77             if self.rq.runq_buildable[task] == 1:
78                 return task
79
80 class RunQueueSchedulerSpeed(RunQueueScheduler):
81     """
82     A scheduler optimised for speed. The priority map is sorted by task weight,
83     heavier weighted tasks (tasks needed by the most other tasks) are run first.
84     """
85     def __init__(self, runqueue):
86         """
87         The priority map is sorted by task weight.
88         """
89         from copy import deepcopy
90
91         self.rq = runqueue
92
93         sortweight = deepcopy(self.rq.runq_weight)
94         sortweight.sort()
95         copyweight = deepcopy(self.rq.runq_weight)
96         self.prio_map = []
97
98         for weight in sortweight:
99             idx = copyweight.index(weight)
100             self.prio_map.append(idx)
101             copyweight[idx] = -1
102
103         self.prio_map.reverse()
104
105 class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
106     """
107     A scheduler optimised to complete .bb files are quickly as possible. The 
108     priority map is sorted by task weight, but then reordered so once a given 
109     .bb file starts to build, its completed as quickly as possible. This works
110     well where disk space is at a premium and classes like OE's rm_work are in 
111     force.
112     """
113     def __init__(self, runqueue):
114         RunQueueSchedulerSpeed.__init__(self, runqueue)
115         from copy import deepcopy
116
117         #FIXME - whilst this groups all fnids together it does not reorder the
118         #fnid groups optimally.
119  
120         basemap = deepcopy(self.prio_map)
121         self.prio_map = []
122         while (len(basemap) > 0):
123             entry = basemap.pop(0)
124             self.prio_map.append(entry)
125             fnid = self.rq.runq_fnid[entry]
126             todel = []
127             for entry in basemap:
128                 entry_fnid = self.rq.runq_fnid[entry]
129                 if entry_fnid == fnid:
130                     todel.append(basemap.index(entry))
131                     self.prio_map.append(entry)
132             todel.reverse()
133             for idx in todel:
134                 del basemap[idx]
135
136 class RunQueue:
137     """
138     BitBake Run Queue implementation
139     """
140     def __init__(self, cooker, cfgData, dataCache, taskData, targets):
141         self.reset_runqueue()
142         self.cooker = cooker
143         self.dataCache = dataCache
144         self.taskData = taskData
145         self.targets = targets
146
147         self.number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1)
148         self.multi_provider_whitelist = (bb.data.getVar("MULTI_PROVIDER_WHITELIST", cfgData) or "").split()
149
150     def reset_runqueue(self):
151
152         self.runq_fnid = []
153         self.runq_task = []
154         self.runq_depends = []
155         self.runq_revdeps = []
156
157     def get_user_idstring(self, task):
158         fn = self.taskData.fn_index[self.runq_fnid[task]]
159         taskname = self.runq_task[task]
160         return "%s, %s" % (fn, taskname)
161
162     def circular_depchains_handler(self, tasks):
163         """
164         Some tasks aren't buildable, likely due to circular dependency issues.
165         Identify the circular dependencies and print them in a user readable format.
166         """
167         from copy import deepcopy
168
169         valid_chains = []
170         explored_deps = {}
171         msgs = []
172
173         def chain_reorder(chain):
174             """
175             Reorder a dependency chain so the lowest task id is first
176             """
177             lowest = 0
178             new_chain = []
179             for entry in range(len(chain)):
180                 if chain[entry] < chain[lowest]:
181                     lowest = entry
182             new_chain.extend(chain[lowest:])
183             new_chain.extend(chain[:lowest])
184             return new_chain
185
186         def chain_compare_equal(chain1, chain2):
187             """
188             Compare two dependency chains and see if they're the same
189             """
190             if len(chain1) != len(chain2):
191                 return False
192             for index in range(len(chain1)):
193                 if chain1[index] != chain2[index]:
194                     return False
195             return True
196             
197         def chain_array_contains(chain, chain_array):
198             """
199             Return True if chain_array contains chain
200             """
201             for ch in chain_array:
202                 if chain_compare_equal(ch, chain):
203                     return True
204             return False
205
206         def find_chains(taskid, prev_chain):
207             prev_chain.append(taskid)
208             total_deps = []
209             total_deps.extend(self.runq_revdeps[taskid])
210             for revdep in self.runq_revdeps[taskid]:
211                 if revdep in prev_chain:
212                     idx = prev_chain.index(revdep)
213                     # To prevent duplicates, reorder the chain to start with the lowest taskid
214                     # and search through an array of those we've already printed
215                     chain = prev_chain[idx:]
216                     new_chain = chain_reorder(chain)
217                     if not chain_array_contains(new_chain, valid_chains):
218                         valid_chains.append(new_chain)
219                         msgs.append("Dependency loop #%d found:\n" % len(valid_chains))
220                         for dep in new_chain:
221                             msgs.append("  Task %s (%s) (depends: %s)\n" % (dep, self.get_user_idstring(dep), self.runq_depends[dep]))
222                         msgs.append("\n")
223                     if len(valid_chains) > 10:
224                         msgs.append("Aborted dependency loops search after 10 matches.\n")
225                         return msgs
226                     continue
227                 scan = False
228                 if revdep not in explored_deps:
229                     scan = True
230                 elif revdep in explored_deps[revdep]:
231                     scan = True
232                 else:
233                     for dep in prev_chain:
234                         if dep in explored_deps[revdep]:
235                             scan = True
236                 if scan:
237                     find_chains(revdep, deepcopy(prev_chain))
238                 for dep in explored_deps[revdep]:
239                     if dep not in total_deps:
240                         total_deps.append(dep)
241
242             explored_deps[taskid] = total_deps
243
244         for task in tasks:
245             find_chains(task, [])
246
247         return msgs
248
249     def calculate_task_weights(self, endpoints):
250         """
251         Calculate a number representing the "weight" of each task. Heavier weighted tasks 
252         have more dependencies and hence should be executed sooner for maximum speed.
253
254         This function also sanity checks the task list finding tasks that its not
255         possible to execute due to circular dependencies.
256         """
257
258         numTasks = len(self.runq_fnid)
259         weight = []
260         deps_left = []
261         task_done = []
262
263         for listid in range(numTasks):
264             task_done.append(False)
265             weight.append(0)
266             deps_left.append(len(self.runq_revdeps[listid]))
267
268         for listid in endpoints:
269             weight[listid] = 1
270             task_done[listid] = True
271
272         while 1:
273             next_points = []
274             for listid in endpoints:
275                 for revdep in self.runq_depends[listid]:
276                     weight[revdep] = weight[revdep] + weight[listid]
277                     deps_left[revdep] = deps_left[revdep] - 1
278                     if deps_left[revdep] == 0:
279                         next_points.append(revdep)
280                         task_done[revdep] = True
281             endpoints = next_points
282             if len(next_points) == 0:
283                 break      
284
285         # Circular dependency sanity check
286         problem_tasks = []
287         for task in range(numTasks):
288             if task_done[task] is False or deps_left[task] != 0:
289                 problem_tasks.append(task)
290                 bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s) is not buildable\n" % (task, self.get_user_idstring(task)))
291                 bb.msg.debug(2, bb.msg.domain.RunQueue, "(Complete marker was %s and the remaining dependency count was %s)\n\n" % (task_done[task], deps_left[task]))
292
293         if problem_tasks:
294             message = "Unbuildable tasks were found.\n"
295             message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n"
296             message = message + "Identifying dependency loops (this may take a short while)...\n"
297             bb.msg.error(bb.msg.domain.RunQueue, message)
298
299             msgs = self.circular_depchains_handler(problem_tasks)
300
301             message = "\n"
302             for msg in msgs:
303                 message = message + msg
304             bb.msg.fatal(bb.msg.domain.RunQueue, message)
305
306         return weight
307
308     def prepare_runqueue(self):
309         """
310         Turn a set of taskData into a RunQueue and compute data needed 
311         to optimise the execution order.
312         """
313
314         depends = []
315         runq_build = []
316
317         taskData = self.taskData
318
319         if len(taskData.tasks_name) == 0:
320             # Nothing to do
321             return
322
323         bb.msg.note(1, bb.msg.domain.RunQueue, "Preparing runqueue")
324
325         # Step A - Work out a list of tasks to run
326         #
327         # Taskdata gives us a list of possible providers for a every target 
328         # ordered by priority (build_targets, run_targets). It also gives
329         # information on each of those providers.
330         #
331         # To create the actual list of tasks to execute we fix the list of 
332         # providers and then resolve the dependencies into task IDs. This 
333         # process is repeated for each type of dependency (tdepends, deptask, 
334         # rdeptast, recrdeptask, idepends).
335
336         for task in range(len(taskData.tasks_name)):
337             fnid = taskData.tasks_fnid[task]
338             fn = taskData.fn_index[fnid]
339             task_deps = self.dataCache.task_deps[fn]
340
341             if fnid not in taskData.failed_fnids:
342
343                 # Resolve task internal dependencies 
344                 #
345                 # e.g. addtask before X after Y
346                 depends = taskData.tasks_tdepends[task]
347
348                 # Resolve 'deptask' dependencies 
349                 #
350                 # e.g. do_sometask[deptask] = "do_someothertask"
351                 # (makes sure sometask runs after someothertask of all DEPENDS)
352                 if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']:
353                     tasknames = task_deps['deptask'][taskData.tasks_name[task]].split()
354                     for depid in taskData.depids[fnid]:
355                         # Won't be in build_targets if ASSUME_PROVIDED
356                         if depid in taskData.build_targets:
357                             depdata = taskData.build_targets[depid][0]
358                             if depdata is not None:
359                                 dep = taskData.fn_index[depdata]
360                                 for taskname in tasknames:
361                                     depends.append(taskData.gettask_id(dep, taskname))
362
363                 # Resolve 'rdeptask' dependencies 
364                 #
365                 # e.g. do_sometask[rdeptask] = "do_someothertask"
366                 # (makes sure sometask runs after someothertask of all RDEPENDS)
367                 if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']:
368                     taskname = task_deps['rdeptask'][taskData.tasks_name[task]]
369                     for depid in taskData.rdepids[fnid]:
370                         if depid in taskData.run_targets:
371                             depdata = taskData.run_targets[depid][0]
372                             if depdata is not None:
373                                 dep = taskData.fn_index[depdata]
374                                 depends.append(taskData.gettask_id(dep, taskname))
375
376                 # Resolve inter-task dependencies 
377                 #
378                 # e.g. do_sometask[depends] = "targetname:do_someothertask"
379                 # (makes sure sometask runs after targetname's someothertask)
380                 idepends = taskData.tasks_idepends[task]
381                 for idepend in idepends:
382                     depid = int(idepend.split(":")[0])
383                     if depid in taskData.build_targets:
384                         # Won't be in build_targets if ASSUME_PROVIDED
385                         depdata = taskData.build_targets[depid][0]
386                         if depdata is not None:
387                             dep = taskData.fn_index[depdata]
388                             depends.append(taskData.gettask_id(dep, idepend.split(":")[1]))
389
390                 def add_recursive_build(depid, depfnid):
391                     """
392                     Add build depends of depid to depends
393                     (if we've not see it before)
394                     (calls itself recursively)
395                     """
396                     if str(depid) in dep_seen:
397                         return
398                     dep_seen.append(depid)
399                     if depid in taskData.build_targets:
400                         depdata = taskData.build_targets[depid][0]
401                         if depdata is not None:
402                             dep = taskData.fn_index[depdata]
403                             idepends = []
404                             # Need to avoid creating new tasks here
405                             taskid = taskData.gettask_id(dep, taskname, False)
406                             if taskid is not None:
407                                 depends.append(taskid)
408                                 fnid = taskData.tasks_fnid[taskid]
409                                 idepends = taskData.tasks_idepends[taskid]
410                                 #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid])
411                             else:
412                                 fnid = taskData.getfn_id(dep)
413                             for nextdepid in taskData.depids[fnid]:
414                                 if nextdepid not in dep_seen:
415                                     add_recursive_build(nextdepid, fnid)
416                             for nextdepid in taskData.rdepids[fnid]:
417                                 if nextdepid not in rdep_seen:
418                                     add_recursive_run(nextdepid, fnid)
419                             for idepend in idepends:
420                                 nextdepid = int(idepend.split(":")[0])
421                                 if nextdepid not in dep_seen:
422                                     add_recursive_build(nextdepid, fnid)
423
424                 def add_recursive_run(rdepid, depfnid):
425                     """
426                     Add runtime depends of rdepid to depends
427                     (if we've not see it before)
428                     (calls itself recursively)
429                     """
430                     if str(rdepid) in rdep_seen:
431                         return
432                     rdep_seen.append(rdepid)
433                     if rdepid in taskData.run_targets:
434                         depdata = taskData.run_targets[rdepid][0]
435                         if depdata is not None:
436                             dep = taskData.fn_index[depdata]
437                             idepends = []
438                             # Need to avoid creating new tasks here
439                             taskid = taskData.gettask_id(dep, taskname, False)
440                             if taskid is not None:
441                                 depends.append(taskid)
442                                 fnid = taskData.tasks_fnid[taskid]
443                                 idepends = taskData.tasks_idepends[taskid]
444                                 #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid])
445                             else:
446                                 fnid = taskData.getfn_id(dep)
447                             for nextdepid in taskData.depids[fnid]:
448                                 if nextdepid not in dep_seen:
449                                     add_recursive_build(nextdepid, fnid)
450                             for nextdepid in taskData.rdepids[fnid]:
451                                 if nextdepid not in rdep_seen:
452                                     add_recursive_run(nextdepid, fnid)
453                             for idepend in idepends:
454                                 nextdepid = int(idepend.split(":")[0])
455                                 if nextdepid not in dep_seen:
456                                     add_recursive_build(nextdepid, fnid)
457
458                 # Resolve recursive 'recrdeptask' dependencies 
459                 #
460                 # e.g. do_sometask[recrdeptask] = "do_someothertask"
461                 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
462                 if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']:
463                     for taskname in task_deps['recrdeptask'][taskData.tasks_name[task]].split():
464                         dep_seen = []
465                         rdep_seen = []
466                         idep_seen = []
467                         for depid in taskData.depids[fnid]:
468                             add_recursive_build(depid, fnid)
469                         for rdepid in taskData.rdepids[fnid]:
470                             add_recursive_run(rdepid, fnid)
471                         for idepend in idepends:
472                             depid = int(idepend.split(":")[0])
473                             add_recursive_build(depid, fnid)
474
475                 # Rmove all self references
476                 if task in depends:
477                     newdep = []
478                     bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends))
479                     for dep in depends:
480                        if task != dep:
481                           newdep.append(dep)
482                     depends = newdep
483
484
485             self.runq_fnid.append(taskData.tasks_fnid[task])
486             self.runq_task.append(taskData.tasks_name[task])
487             self.runq_depends.append(Set(depends))
488             self.runq_revdeps.append(Set())
489
490             runq_build.append(0)
491
492         # Step B - Mark all active tasks
493         #
494         # Start with the tasks we were asked to run and mark all dependencies
495         # as active too. If the task is to be 'forced', clear its stamp. Once
496         # all active tasks are marked, prune the ones we don't need.
497
498         bb.msg.note(2, bb.msg.domain.RunQueue, "Marking Active Tasks")
499
500         def mark_active(listid, depth):
501             """
502             Mark an item as active along with its depends
503             (calls itself recursively)
504             """
505
506             if runq_build[listid] == 1:
507                 return
508
509             runq_build[listid] = 1
510
511             depends = self.runq_depends[listid]
512             for depend in depends:
513                 mark_active(depend, depth+1)
514
515         for target in self.targets:
516             targetid = taskData.getbuild_id(target[0])
517
518             if targetid not in taskData.build_targets:
519                 continue
520
521             if targetid in taskData.failed_deps:
522                 continue
523
524             fnid = taskData.build_targets[targetid][0]
525
526             # Remove stamps for targets if force mode active
527             if self.cooker.configuration.force:
528                 fn = taskData.fn_index[fnid]
529                 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (target[1], fn))
530                 bb.build.del_stamp(target[1], self.dataCache, fn)
531
532             if fnid in taskData.failed_fnids:
533                 continue
534
535             if target[1] not in taskData.tasks_lookup[fnid]:
536                 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s does not exist for target %s" % (target[1], target[0]))
537
538             listid = taskData.tasks_lookup[fnid][target[1]]
539
540             mark_active(listid, 1)
541
542         # Step C - Prune all inactive tasks
543         #
544         # Once all active tasks are marked, prune the ones we don't need.
545
546         maps = []
547         delcount = 0
548         for listid in range(len(self.runq_fnid)):
549             if runq_build[listid-delcount] == 1:
550                 maps.append(listid-delcount)
551             else:
552                 del self.runq_fnid[listid-delcount]
553                 del self.runq_task[listid-delcount]
554                 del self.runq_depends[listid-delcount]
555                 del runq_build[listid-delcount]
556                 del self.runq_revdeps[listid-delcount]
557                 delcount = delcount + 1
558                 maps.append(-1)
559
560         #
561         # Step D - Sanity checks and computation
562         #
563
564         # Check to make sure we still have tasks to run
565         if len(self.runq_fnid) == 0:
566             if not taskData.abort:
567                 bb.msg.fatal(bb.msg.domain.RunQueue, "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.")
568             else:               
569                 bb.msg.fatal(bb.msg.domain.RunQueue, "No active tasks and not in --continue mode?! Please report this bug.")
570
571         bb.msg.note(2, bb.msg.domain.RunQueue, "Pruned %s inactive tasks, %s left" % (delcount, len(self.runq_fnid)))
572
573         # Remap the dependencies to account for the deleted tasks
574         # Check we didn't delete a task we depend on
575         for listid in range(len(self.runq_fnid)):
576             newdeps = []
577             origdeps = self.runq_depends[listid]
578             for origdep in origdeps:
579                 if maps[origdep] == -1:
580                     bb.msg.fatal(bb.msg.domain.RunQueue, "Invalid mapping - Should never happen!")
581                 newdeps.append(maps[origdep])
582             self.runq_depends[listid] = Set(newdeps)
583
584         bb.msg.note(2, bb.msg.domain.RunQueue, "Assign Weightings")
585
586         # Generate a list of reverse dependencies to ease future calculations
587         for listid in range(len(self.runq_fnid)):
588             for dep in self.runq_depends[listid]:
589                 self.runq_revdeps[dep].add(listid)
590
591         # Identify tasks at the end of dependency chains
592         # Error on circular dependency loops (length two)
593         endpoints = []
594         for listid in range(len(self.runq_fnid)):
595             revdeps = self.runq_revdeps[listid]
596             if len(revdeps) == 0:
597                 endpoints.append(listid)
598             for dep in revdeps:
599                 if dep in self.runq_depends[listid]:
600                     #self.dump_data(taskData)
601                     bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep] , taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid]))
602
603         bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints))
604
605
606         # Calculate task weights 
607         # Check of higher length circular dependencies
608         self.runq_weight = self.calculate_task_weights(endpoints)
609
610         # Decide what order to execute the tasks in, pick a scheduler
611         # FIXME - Allow user selection
612         #self.sched = RunQueueScheduler(self)
613         self.sched = RunQueueSchedulerSpeed(self)
614         #self.sched = RunQueueSchedulerCompletion(self)
615
616         # Sanity Check - Check for multiple tasks building the same provider
617         prov_list = {}
618         seen_fn = []
619         for task in range(len(self.runq_fnid)):
620             fn = taskData.fn_index[self.runq_fnid[task]]
621             if fn in seen_fn:
622                 continue
623             seen_fn.append(fn)
624             for prov in self.dataCache.fn_provides[fn]:
625                 if prov not in prov_list:
626                     prov_list[prov] = [fn]
627                 elif fn not in prov_list[prov]: 
628                     prov_list[prov].append(fn)
629         error = False
630         for prov in prov_list:
631             if len(prov_list[prov]) > 1 and prov not in self.multi_provider_whitelist:
632                 error = True
633                 bb.msg.error(bb.msg.domain.RunQueue, "Multiple .bb files are due to be built which each provide %s (%s).\n This usually means one provides something the other doesn't and should." % (prov, " ".join(prov_list[prov])))
634         #if error:
635         #    bb.msg.fatal(bb.msg.domain.RunQueue, "Corrupted metadata configuration detected, aborting...")
636
637         #self.dump_data(taskData)
638
639     def execute_runqueue(self):
640         """
641         Run the tasks in a queue prepared by prepare_runqueue
642         Upon failure, optionally try to recover the build using any alternate providers
643         (if the abort on failure configuration option isn't set)
644         """
645
646         failures = 0
647         while 1:
648             failed_fnids = []
649             try:
650                 self.execute_runqueue_internal()
651             finally:
652                 if self.master_process:
653                     failed_fnids = self.finish_runqueue()
654             if len(failed_fnids) == 0:
655                 return failures
656             if self.taskData.abort:
657                 raise bb.runqueue.TaskFailure(failed_fnids)
658             for fnid in failed_fnids:
659                 #print "Failure: %s %s %s" % (fnid, self.taskData.fn_index[fnid],  self.runq_task[fnid])
660                 self.taskData.fail_fnid(fnid)
661                 failures = failures + 1
662             self.reset_runqueue()
663             self.prepare_runqueue()
664
665     def execute_runqueue_initVars(self):
666
667         self.stats = RunQueueStats()
668
669         self.active_builds = 0
670         self.runq_buildable = []
671         self.runq_running = []
672         self.runq_complete = []
673         self.build_pids = {}
674         self.failed_fnids = []
675         self.master_process = True
676
677         # Mark initial buildable tasks
678         for task in range(len(self.runq_fnid)):
679             self.runq_running.append(0)
680             self.runq_complete.append(0)
681             if len(self.runq_depends[task]) == 0:
682                 self.runq_buildable.append(1)
683             else:
684                 self.runq_buildable.append(0)
685
686     def task_complete(self, task):
687         """
688         Mark a task as completed
689         Look at the reverse dependencies and mark any task with 
690         completed dependencies as buildable
691         """
692         self.runq_complete[task] = 1
693         for revdep in self.runq_revdeps[task]:
694             if self.runq_running[revdep] == 1:
695                 continue
696             if self.runq_buildable[revdep] == 1:
697                 continue
698             alldeps = 1
699             for dep in self.runq_depends[revdep]:
700                 if self.runq_complete[dep] != 1:
701                     alldeps = 0
702             if alldeps == 1:
703                 self.runq_buildable[revdep] = 1
704                 fn = self.taskData.fn_index[self.runq_fnid[revdep]]
705                 taskname = self.runq_task[revdep]
706                 bb.msg.debug(1, bb.msg.domain.RunQueue, "Marking task %s (%s, %s) as buildable" % (revdep, fn, taskname))
707
708     def execute_runqueue_internal(self):
709         """
710         Run the tasks in a queue prepared by prepare_runqueue
711         """
712
713         bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
714
715         self.execute_runqueue_initVars()
716
717         if len(self.runq_fnid) == 0:
718             # nothing to do
719             return []
720
721         def sigint_handler(signum, frame):
722             raise KeyboardInterrupt
723
724         # RP - this code allows tasks to run out of the correct order - disabled, FIXME
725         # Find any tasks with current stamps and remove them from the queue
726         #for task1 in range(len(self.runq_fnid)):
727         #    task = self.prio_map[task1]
728         #    fn = self.taskData.fn_index[self.runq_fnid[task]]
729         #    taskname = self.runq_task[task]
730         #    if bb.build.stamp_is_current(taskname, self.dataCache, fn):
731         #        bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
732         #        self.runq_running[task] = 1
733         #        self.task_complete(task)
734         #        self.stats.taskCompleted()
735         #        self.stats.taskSkipped()
736
737         while True:
738             task = self.sched.next()
739             if task is not None:
740                 fn = self.taskData.fn_index[self.runq_fnid[task]]
741
742                 taskname = self.runq_task[task]
743                 if bb.build.stamp_is_current(taskname, self.dataCache, fn):
744                     bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
745                     self.runq_running[task] = 1
746                     self.task_complete(task)
747                     self.stats.taskCompleted()
748                     self.stats.taskSkipped()
749                     continue
750
751                 bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task)))
752                 try: 
753                     pid = os.fork() 
754                 except OSError, e: 
755                     bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
756                 if pid == 0:
757                     # Bypass master process' handling
758                     self.master_process = False
759                     # Stop Ctrl+C being sent to children
760                     # signal.signal(signal.SIGINT, signal.SIG_IGN)
761                     # Make the child the process group leader
762                     os.setpgid(0, 0)
763                     newsi = os.open('/dev/null', os.O_RDWR)
764                     os.dup2(newsi, sys.stdin.fileno())
765                     self.cooker.configuration.cmd = taskname[3:]
766                     try: 
767                         self.cooker.tryBuild(fn, False)
768                     except bb.build.EventException:
769                         bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
770                         sys.exit(1)
771                     except:
772                         bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
773                         raise
774                     sys.exit(0)
775                 self.build_pids[pid] = task
776                 self.runq_running[task] = 1
777                 self.active_builds = self.active_builds + 1
778                 if self.active_builds < self.number_tasks:
779                     continue
780             if self.active_builds > 0:
781                 result = os.waitpid(-1, 0)
782                 self.active_builds = self.active_builds - 1
783                 task = self.build_pids[result[0]]
784                 if result[1] != 0:
785                     del self.build_pids[result[0]]
786                     bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task)))
787                     self.failed_fnids.append(self.runq_fnid[task])
788                     self.stats.taskFailed()
789                     break
790                 self.task_complete(task)
791                 self.stats.taskCompleted()
792                 del self.build_pids[result[0]]
793                 continue
794             return
795
796     def finish_runqueue(self):
797         try:
798             while self.active_builds > 0:
799                 bb.msg.note(1, bb.msg.domain.RunQueue, "Waiting for %s active tasks to finish" % self.active_builds)
800                 tasknum = 1
801                 for k, v in self.build_pids.iteritems():
802                      bb.msg.note(1, bb.msg.domain.RunQueue, "%s: %s (%s)" % (tasknum, self.get_user_idstring(v), k))
803                      tasknum = tasknum + 1
804                 result = os.waitpid(-1, 0)
805                 task = self.build_pids[result[0]]
806                 if result[1] != 0:
807                      bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task)))
808                      self.failed_fnids.append(self.runq_fnid[task])
809                      self.stats.taskFailed()
810                 del self.build_pids[result[0]]
811                 self.active_builds = self.active_builds - 1
812             bb.msg.note(1, bb.msg.domain.RunQueue, "Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed." % (self.stats.completed, self.stats.skipped, self.stats.failed))
813             return self.failed_fnids
814         except KeyboardInterrupt:
815             bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.active_builds)
816             for k, v in self.build_pids.iteritems():
817                  try:
818                      os.kill(-k, signal.SIGINT)
819                  except:
820                      pass
821             raise
822
823         # Sanity Checks
824         for task in range(len(self.runq_fnid)):
825             if self.runq_buildable[task] == 0:
826                 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never buildable!" % task)
827             if self.runq_running[task] == 0:
828                 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never ran!" % task)
829             if self.runq_complete[task] == 0:
830                 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never completed!" % task)
831
832         bb.msg.note(1, bb.msg.domain.RunQueue, "Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed." % (self.stats.completed, self.stats.skipped, self.stats.failed))
833
834         return self.failed_fnids
835
836     def dump_data(self, taskQueue):
837         """
838         Dump some debug information on the internal data structures
839         """
840         bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:")
841         for task in range(len(self.runq_fnid)):
842                 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task, 
843                         taskQueue.fn_index[self.runq_fnid[task]], 
844                         self.runq_task[task], 
845                         self.runq_weight[task], 
846                         self.runq_depends[task], 
847                         self.runq_revdeps[task]))
848
849         bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:")
850         for task1 in range(len(self.runq_fnid)):
851             if task1 in self.prio_map:
852                 task = self.prio_map[task1]
853                 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task, 
854                         taskQueue.fn_index[self.runq_fnid[task]], 
855                         self.runq_task[task], 
856                         self.runq_weight[task], 
857                         self.runq_depends[task], 
858                         self.runq_revdeps[task]))