Merge the BBCLASSEXTEND code from Poky. This allows once recipe to provide mutliple...
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006 - 2007 Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from bb import xmlrpcserver, command
29 import itertools, sre_constants
30
31 class MultipleMatches(Exception):
32     """
33     Exception raised when multiple file matches are found
34     """
35
36 class ParsingErrorsFound(Exception):
37     """
38     Exception raised when parsing errors are found
39     """
40
41 class NothingToBuild(Exception):
42     """
43     Exception raised when there is nothing to build
44     """
45
46
47 # Different states cooker can be in
48 cookerClean = 1
49 cookerParsed = 2
50
51 # Different action states the cooker can be in
52 cookerRun = 1           # Cooker is running normally
53 cookerShutdown = 2      # Active tasks should be brought to a controlled stop
54 cookerStop = 3          # Stop, now!
55
56 #============================================================================#
57 # BBCooker
58 #============================================================================#
59 class BBCooker:
60     """
61     Manages one bitbake build run
62     """
63
64     def __init__(self, configuration):
65         self.status = None
66
67         self.cache = None
68         self.bb_cache = None
69
70         self.server = bb.xmlrpcserver.BitBakeXMLRPCServer(self)
71         #self.server.register_function(self.showEnvironment)
72
73         self.configuration = configuration
74
75         if self.configuration.verbose:
76             bb.msg.set_verbose(True)
77
78         if self.configuration.debug:
79             bb.msg.set_debug_level(self.configuration.debug)
80         else:
81             bb.msg.set_debug_level(0)
82
83         if self.configuration.debug_domains:
84             bb.msg.set_debug_domains(self.configuration.debug_domains)
85
86         self.configuration.data = bb.data.init()
87
88         bb.data.inheritFromOS(self.configuration.data)
89
90         for f in self.configuration.file:
91             self.parseConfigurationFile( f )
92
93         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
94
95         if not self.configuration.cmd:
96             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
97
98         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
99         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
100             self.configuration.pkgs_to_build.extend(bbpkgs.split())
101
102         #
103         # Special updated configuration we use for firing events
104         #
105         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
106         bb.data.update_data(self.configuration.event_data)
107
108         # TOSTOP must not be set or our children will hang when they output
109         fd = sys.stdout.fileno()
110         if os.isatty(fd):
111             import termios
112             tcattr = termios.tcgetattr(fd)
113             if tcattr[3] & termios.TOSTOP:
114                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
115                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
116                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
117
118         self.command = bb.command.Command(self)
119         self.cookerIdle = True
120         self.cookerState = cookerClean
121         self.cookerAction = cookerRun
122         self.server.register_idle_function(self.runCommands, self)
123
124     def parseConfiguration(self):
125
126
127         # Change nice level if we're asked to
128         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
129         if nice:
130             curnice = os.nice(0)
131             nice = int(nice) - curnice
132             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
133
134     def parseCommandLine(self):
135         # Parse any commandline into actions
136         if self.configuration.show_environment:
137             self.commandlineAction = None
138
139             if 'world' in self.configuration.pkgs_to_build:
140                 bb.error("'world' is not a valid target for --environment.")
141             elif len(self.configuration.pkgs_to_build) > 1:
142                 bb.error("Only one target can be used with the --environment option.")
143             elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
144                 bb.error("No target should be used with the --environment and --buildfile options.")
145             else:
146                 self.commandlineAction = ["showEnvironment", self.configuration.buildfile, self.configuration.pkgs_to_build]
147         elif self.configuration.buildfile is not None:
148             self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
149         elif self.configuration.show_versions:
150             self.commandlineAction = ["showVersions"]
151         elif self.configuration.parse_only:
152             self.commandlineAction = ["parseFiles"]
153         # FIXME - implement
154         #elif self.configuration.interactive:
155         #    self.interactiveMode()
156         elif self.configuration.dot_graph:
157             if self.configuration.pkgs_to_build:
158                 self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
159             else:
160                 self.commandlineAction = None
161                 bb.error("Please specify a package name for dependency graph generation.")
162         else:
163             if self.configuration.pkgs_to_build:
164                 self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
165             else:
166                 self.commandlineAction = None
167                 bb.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
168
169     def runCommands(self, server, data, abort):
170         """
171         Run any queued asynchronous command
172         This is done by the idle handler so it runs in true context rather than
173         tied to any UI.
174         """
175         if self.cookerIdle and not abort:
176             self.command.runAsyncCommand()
177
178         # Always reschedule
179         return True
180
181     def tryBuildPackage(self, fn, item, task, the_data):
182         """
183         Build one task of a package, optionally build following task depends
184         """
185         try:
186             if not self.configuration.dry_run:
187                 bb.build.exec_task('do_%s' % task, the_data)
188             return True
189         except bb.build.FuncFailed:
190             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
191             raise
192         except bb.build.EventException, e:
193             event = e.args[1]
194             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
195             raise
196
197     def tryBuild(self, fn, task):
198         """
199         Build a provider and its dependencies. 
200         build_depends is a list of previous build dependencies (not runtime)
201         If build_depends is empty, we're dealing with a runtime depends
202         """
203
204         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
205
206         item = self.status.pkg_fn[fn]
207
208         #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
209         #    return True
210
211         return self.tryBuildPackage(fn, item, task, the_data)
212
213     def showVersions(self):
214
215         # Need files parsed
216         self.updateCache()
217
218         pkg_pn = self.status.pkg_pn
219         preferred_versions = {}
220         latest_versions = {}
221
222         # Sort by priority
223         for pn in pkg_pn.keys():
224             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
225             preferred_versions[pn] = (pref_ver, pref_file)
226             latest_versions[pn] = (last_ver, last_file)
227
228         pkg_list = pkg_pn.keys()
229         pkg_list.sort()
230
231         bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version"))
232         bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "================="))
233
234         for p in pkg_list:
235             pref = preferred_versions[p]
236             latest = latest_versions[p]
237
238             prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
239             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
240
241             if pref == latest:
242                 prefstr = ""
243
244             bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr))
245
246     def showEnvironment(self, buildfile = None, pkgs_to_build = []):
247         """
248         Show the outer or per-package environment
249         """
250         fn = None
251         envdata = None
252
253         if buildfile:
254             self.cb = None
255             self.bb_cache = bb.cache.init(self)
256             fn = self.matchFile(buildfile)
257         elif len(pkgs_to_build) == 1:
258             self.updateCache()
259
260             localdata = data.createCopy(self.configuration.data)
261             bb.data.update_data(localdata)
262             bb.data.expandKeys(localdata)
263
264             taskdata = bb.taskdata.TaskData(self.configuration.abort)
265             taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
266             taskdata.add_unresolved(localdata, self.status)
267
268             targetid = taskdata.getbuild_id(pkgs_to_build[0])
269             fnid = taskdata.build_targets[targetid][0]
270             fn = taskdata.fn_index[fnid]
271         else:
272             envdata = self.configuration.data
273
274         if fn:
275             try:
276                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
277             except IOError, e:
278                 bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
279                 raise
280             except Exception, e:
281                 bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
282                 raise
283
284         class dummywrite:
285             def __init__(self):
286                 self.writebuf = ""
287             def write(self, output):
288                 self.writebuf = self.writebuf + output
289
290         # emit variables and shell functions
291         try:
292             data.update_data(envdata)
293             wb = dummywrite()
294             data.emit_env(wb, envdata, True)
295             bb.msg.plain(wb.writebuf)
296         except Exception, e:
297             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
298         # emit the metadata which isnt valid shell
299         data.expandKeys(envdata)
300         for e in envdata.keys():
301             if data.getVarFlag( e, 'python', envdata ):
302                 bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
303
304     def generateDepTreeData(self, pkgs_to_build, task):
305         """
306         Create a dependency tree of pkgs_to_build, returning the data.
307         """
308
309         # Need files parsed
310         self.updateCache()
311
312         # If we are told to do the None task then query the default task
313         if (task == None):
314             task = self.configuration.cmd
315
316         pkgs_to_build = self.checkPackages(pkgs_to_build)
317
318         localdata = data.createCopy(self.configuration.data)
319         bb.data.update_data(localdata)
320         bb.data.expandKeys(localdata)
321         taskdata = bb.taskdata.TaskData(self.configuration.abort)
322
323         runlist = []
324         for k in pkgs_to_build:
325             taskdata.add_provider(localdata, self.status, k)
326             runlist.append([k, "do_%s" % task])
327         taskdata.add_unresolved(localdata, self.status)
328
329         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
330         rq.prepare_runqueue()
331
332         seen_fnids = []  
333         depend_tree = {}
334         depend_tree["depends"] = {}
335         depend_tree["tdepends"] = {}
336         depend_tree["pn"] = {}
337         depend_tree["rdepends-pn"] = {}
338         depend_tree["packages"] = {}
339         depend_tree["rdepends-pkg"] = {}
340         depend_tree["rrecs-pkg"] = {}
341
342         for task in range(len(rq.runq_fnid)):
343             taskname = rq.runq_task[task]
344             fnid = rq.runq_fnid[task]
345             fn = taskdata.fn_index[fnid]
346             pn = self.status.pkg_fn[fn]
347             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
348             if pn not in depend_tree["pn"]:
349                 depend_tree["pn"][pn] = {}
350                 depend_tree["pn"][pn]["filename"] = fn
351                 depend_tree["pn"][pn]["version"] = version
352             for dep in rq.runq_depends[task]:
353                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
354                 deppn = self.status.pkg_fn[depfn]
355                 dotname = "%s.%s" % (pn, rq.runq_task[task])
356                 if not dotname in depend_tree["tdepends"]:
357                     depend_tree["tdepends"][dotname] = []
358                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep]))
359             if fnid not in seen_fnids:
360                 seen_fnids.append(fnid)
361                 packages = []
362
363                 depend_tree["depends"][pn] = []
364                 for dep in taskdata.depids[fnid]:
365                     depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
366
367                 depend_tree["rdepends-pn"][pn] = []
368                 for rdep in taskdata.rdepids[fnid]:
369                         depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
370
371                 rdepends = self.status.rundeps[fn]
372                 for package in rdepends:
373                     depend_tree["rdepends-pkg"][package] = []
374                     for rdepend in rdepends[package]:
375                         depend_tree["rdepends-pkg"][package].append(rdepend)
376                     packages.append(package)
377
378                 rrecs = self.status.runrecs[fn]
379                 for package in rrecs:
380                     depend_tree["rrecs-pkg"][package] = []
381                     for rdepend in rrecs[package]:
382                         depend_tree["rrecs-pkg"][package].append(rdepend)
383                     if not package in packages:
384                         packages.append(package)
385
386                 for package in packages:
387                     if package not in depend_tree["packages"]:
388                         depend_tree["packages"][package] = {}
389                         depend_tree["packages"][package]["pn"] = pn
390                         depend_tree["packages"][package]["filename"] = fn
391                         depend_tree["packages"][package]["version"] = version
392
393         return depend_tree
394
395
396     def generateDepTreeEvent(self, pkgs_to_build, task):
397         """
398         Create a task dependency graph of pkgs_to_build.
399         Generate an event with the result
400         """
401         depgraph = self.generateDepTreeData(pkgs_to_build, task)
402         bb.event.fire(bb.event.DepTreeGenerated(self.configuration.data, depgraph))
403
404     def generateDotGraphFiles(self, pkgs_to_build, task):
405         """
406         Create a task dependency graph of pkgs_to_build.
407         Save the result to a set of .dot files.
408         """
409
410         depgraph = self.generateDepTreeData(pkgs_to_build, task)
411
412         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
413         depends_file = file('pn-depends.dot', 'w' )
414         print >> depends_file, "digraph depends {"
415         for pn in depgraph["pn"]:
416             fn = depgraph["pn"][pn]["filename"]
417             version = depgraph["pn"][pn]["version"]
418             print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
419         for pn in depgraph["depends"]:
420             for depend in depgraph["depends"][pn]:
421                 print >> depends_file, '"%s" -> "%s"' % (pn, depend)
422         for pn in depgraph["rdepends-pn"]:
423             for rdepend in depgraph["rdepends-pn"][pn]:
424                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend)
425         print >> depends_file,  "}"
426         bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
427
428         depends_file = file('package-depends.dot', 'w' )
429         print >> depends_file, "digraph depends {"
430         for package in depgraph["packages"]:
431             pn = depgraph["packages"][package]["pn"]
432             fn = depgraph["packages"][package]["filename"]
433             version = depgraph["packages"][package]["version"]
434             if package == pn:
435                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
436             else:
437                 print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
438             for depend in depgraph["depends"][pn]:
439                 print >> depends_file, '"%s" -> "%s"' % (package, depend)
440         for package in depgraph["rdepends-pkg"]:
441             for rdepend in depgraph["rdepends-pkg"][package]:
442                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
443         for package in depgraph["rrecs-pkg"]:
444             for rdepend in depgraph["rrecs-pkg"][package]:
445                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
446         print >> depends_file,  "}"
447         bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
448
449         tdepends_file = file('task-depends.dot', 'w' )
450         print >> tdepends_file, "digraph depends {"
451         for task in depgraph["tdepends"]:
452             (pn, taskname) = task.rsplit(".", 1)
453             fn = depgraph["pn"][pn]["filename"]
454             version = depgraph["pn"][pn]["version"]
455             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
456             for dep in depgraph["tdepends"][task]:
457                 print >> tdepends_file, '"%s" -> "%s"' % (task, dep)
458         print >> tdepends_file,  "}"
459         bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
460
461     def buildDepgraph( self ):
462         all_depends = self.status.all_depends
463         pn_provides = self.status.pn_provides
464
465         localdata = data.createCopy(self.configuration.data)
466         bb.data.update_data(localdata)
467         bb.data.expandKeys(localdata)
468
469         def calc_bbfile_priority(filename):
470             for (regex, pri) in self.status.bbfile_config_priorities:
471                 if regex.match(filename):
472                     return pri
473             return 0
474
475         # Handle PREFERRED_PROVIDERS
476         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
477             try:
478                 (providee, provider) = p.split(':')
479             except:
480                 bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
481                 continue
482             if providee in self.status.preferred and self.status.preferred[providee] != provider:
483                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
484             self.status.preferred[providee] = provider
485
486         # Calculate priorities for each file
487         for p in self.status.pkg_fn.keys():
488             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
489
490     def buildWorldTargetList(self):
491         """
492          Build package list for "bitbake world"
493         """
494         all_depends = self.status.all_depends
495         pn_provides = self.status.pn_provides
496         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
497         for f in self.status.possible_world:
498             terminal = True
499             pn = self.status.pkg_fn[f]
500
501             for p in pn_provides[pn]:
502                 if p.startswith('virtual/'):
503                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
504                     terminal = False
505                     break
506                 for pf in self.status.providers[p]:
507                     if self.status.pkg_fn[pf] != pn:
508                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
509                         terminal = False
510                         break
511             if terminal:
512                 self.status.world_target.add(pn)
513
514             # drop reference count now
515             self.status.possible_world = None
516             self.status.all_depends    = None
517
518     def interactiveMode( self ):
519         """Drop off into a shell"""
520         try:
521             from bb import shell
522         except ImportError, details:
523             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
524         else:
525             shell.start( self )
526
527     def parseConfigurationFile( self, afile ):
528         try:
529             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
530
531             # Handle any INHERITs and inherit the base class
532             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
533             for inherit in inherits:
534                 self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
535
536             # Nomally we only register event handlers at the end of parsing .bb files
537             # We register any handlers we've found so far here...
538             for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
539                 bb.event.register(var,bb.data.getVar(var, self.configuration.data))
540
541             bb.fetch.fetcher_init(self.configuration.data)
542
543             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
544
545         except IOError, e:
546             bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (afile, str(e)))
547         except bb.parse.ParseError, details:
548             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
549
550     def handleCollections( self, collections ):
551         """Handle collections"""
552         if collections:
553             collection_list = collections.split()
554             for c in collection_list:
555                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
556                 if regex == None:
557                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
558                     continue
559                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
560                 if priority == None:
561                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
562                     continue
563                 try:
564                     cre = re.compile(regex)
565                 except re.error:
566                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
567                     continue
568                 try:
569                     pri = int(priority)
570                     self.status.bbfile_config_priorities.append((cre, pri))
571                 except ValueError:
572                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
573
574     def buildSetVars(self):
575         """
576         Setup any variables needed before starting a build
577         """
578         if not bb.data.getVar("BUILDNAME", self.configuration.data):
579             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
580         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data)
581
582     def matchFiles(self, buildfile):
583         """
584         Find the .bb files which match the expression in 'buildfile'.
585         """
586
587         bf = os.path.abspath(buildfile)
588         try:
589             os.stat(bf)
590             return [bf]
591         except OSError:
592             (filelist, masked) = self.collect_bbfiles()
593             regexp = re.compile(buildfile)
594             matches = []
595             for f in filelist:
596                 if regexp.search(f) and os.path.isfile(f):
597                     bf = f
598                     matches.append(f)
599             return matches
600
601     def matchFile(self, buildfile):
602         """
603         Find the .bb file which matches the expression in 'buildfile'.
604         Raise an error if multiple files
605         """
606         matches = self.matchFiles(buildfile)
607         if len(matches) != 1:
608             bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
609             for f in matches:
610                 bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
611             raise MultipleMatches
612         return matches[0]
613
614     def buildFile(self, buildfile, task):
615         """
616         Build the file matching regexp buildfile
617         """
618
619         # Parse the configuration here. We need to do it explicitly here since
620         # buildFile() doesn't use the cache
621         self.parseConfiguration()
622
623         # If we are told to do the None task then query the default task
624         if (task == None):
625             task = self.configuration.cmd
626
627         fn = self.matchFile(buildfile)
628         self.buildSetVars()
629
630         # Load data into the cache for fn and parse the loaded cache data
631         self.bb_cache = bb.cache.init(self)
632         self.status = bb.cache.CacheData()
633         self.bb_cache.loadData(fn, self.configuration.data, self.status)      
634
635         # Tweak some variables
636         item = self.bb_cache.getVar('PN', fn, True)
637         self.status.ignored_dependencies = set()
638         self.status.bbfile_priority[fn] = 1
639
640         # Remove external dependencies
641         self.status.task_deps[fn]['depends'] = {}
642         self.status.deps[fn] = []
643         self.status.rundeps[fn] = []
644         self.status.runrecs[fn] = []
645
646         # Remove stamp for target if force mode active
647         if self.configuration.force:
648             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn))
649             bb.build.del_stamp('do_%s' % task, self.status, fn)
650
651         # Setup taskdata structure
652         taskdata = bb.taskdata.TaskData(self.configuration.abort)
653         taskdata.add_provider(self.configuration.data, self.status, item)
654
655         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
656         bb.event.fire(bb.event.BuildStarted(buildname, [item], self.configuration.event_data))
657
658         # Execute the runqueue
659         runlist = [[item, "do_%s" % task]]
660
661         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
662
663         def buildFileIdle(server, rq, abort):
664
665             if abort or self.cookerAction == cookerStop:
666                 rq.finish_runqueue(True)
667             elif self.cookerAction == cookerShutdown:
668                 rq.finish_runqueue(False)
669             failures = 0
670             try:
671                 retval = rq.execute_runqueue()
672             except runqueue.TaskFailure, fnids:
673                 for fnid in fnids:
674                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
675                     failures = failures + 1
676                 retval = False
677             if not retval:
678                 self.cookerIdle = True
679                 self.command.finishAsyncCommand()
680                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
681             return retval
682
683         self.cookerIdle = False
684         self.server.register_idle_function(buildFileIdle, rq)
685
686     def buildTargets(self, targets, task):
687         """
688         Attempt to build the targets specified
689         """
690
691         # Need files parsed
692         self.updateCache()
693
694         # If we are told to do the NULL task then query the default task
695         if (task == None):
696             task = self.configuration.cmd
697
698         targets = self.checkPackages(targets)
699
700         def buildTargetsIdle(server, rq, abort):
701
702             if abort or self.cookerAction == cookerStop:
703                 rq.finish_runqueue(True)
704             elif self.cookerAction == cookerShutdown:
705                 rq.finish_runqueue(False)
706             failures = 0
707             try:
708                 retval = rq.execute_runqueue()
709             except runqueue.TaskFailure, fnids:
710                 for fnid in fnids:
711                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
712                     failures = failures + 1
713                 retval = False
714             if not retval:
715                 self.cookerIdle = True
716                 self.command.finishAsyncCommand()
717                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
718             return retval
719
720         self.buildSetVars()
721
722         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
723         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
724
725         localdata = data.createCopy(self.configuration.data)
726         bb.data.update_data(localdata)
727         bb.data.expandKeys(localdata)
728
729         taskdata = bb.taskdata.TaskData(self.configuration.abort)
730
731         runlist = []
732         for k in targets:
733             taskdata.add_provider(localdata, self.status, k)
734             runlist.append([k, "do_%s" % task])
735         taskdata.add_unresolved(localdata, self.status)
736
737         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
738
739         self.cookerIdle = False
740         self.server.register_idle_function(buildTargetsIdle, rq)
741
742     def updateCache(self):
743
744         self.parseConfiguration ()
745         if self.cookerState == cookerParsed:
746             return
747
748         # Import Psyco if available and not disabled
749         import platform
750         if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
751             if not self.configuration.disable_psyco:
752                 try:
753                     import psyco
754                 except ImportError:
755                     bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
756                 else:
757                     psyco.bind( self.parse_bbfiles )
758             else:
759                 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
760
761         self.status = bb.cache.CacheData()
762
763         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
764         self.status.ignored_dependencies = set(ignore.split())
765
766         for dep in self.configuration.extra_assume_provided:
767             self.status.ignored_dependencies.add(dep)
768
769         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
770
771         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
772         (filelist, masked) = self.collect_bbfiles()
773         bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
774         self.parse_bbfiles(filelist, masked)
775         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
776
777         self.buildDepgraph()
778
779         self.cookerState = cookerParsed
780
781     def checkPackages(self, pkgs_to_build):
782
783         if len(pkgs_to_build) == 0:
784             raise NothingToBuild
785
786         if 'world' in pkgs_to_build:
787             self.buildWorldTargetList()
788             pkgs_to_build.remove('world')
789             for t in self.status.world_target:
790                 pkgs_to_build.append(t)
791
792         return pkgs_to_build
793
794     def get_bbfiles( self, path = os.getcwd() ):
795         """Get list of default .bb files by reading out the current directory"""
796         contents = os.listdir(path)
797         bbfiles = []
798         for f in contents:
799             (root, ext) = os.path.splitext(f)
800             if ext == ".bb":
801                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
802         return bbfiles
803
804     def find_bbfiles( self, path ):
805         """Find all the .bb files in a directory"""
806         from os.path import join
807
808         found = []
809         for dir, dirs, files in os.walk(path):
810             for ignored in ('SCCS', 'CVS', '.svn'):
811                 if ignored in dirs:
812                     dirs.remove(ignored)
813             found += [join(dir,f) for f in files if f.endswith('.bb')]
814
815         return found
816
817     def collect_bbfiles( self ):
818         """Collect all available .bb build files"""
819         parsed, cached, skipped, masked = 0, 0, 0, 0
820         self.bb_cache = bb.cache.init(self)
821
822         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
823         data.setVar("BBFILES", " ".join(files), self.configuration.data)
824
825         if not len(files):
826             files = self.get_bbfiles()
827
828         if not len(files):
829             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
830
831         newfiles = []
832         for f in files:
833             if os.path.isdir(f):
834                 dirfiles = self.find_bbfiles(f)
835                 if dirfiles:
836                     newfiles += dirfiles
837                     continue
838             else:
839                 globbed = glob.glob(f)
840                 if not globbed and os.path.exists(f):
841                     globbed = [f]
842                 newfiles += globbed
843
844         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
845
846         if not bbmask:
847             return (newfiles, 0)
848
849         try:
850             bbmask_compiled = re.compile(bbmask)
851         except sre_constants.error:
852             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
853
854         finalfiles = []
855         for f in newfiles:
856             if bbmask_compiled.search(f):
857                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
858                 masked += 1
859                 continue
860             finalfiles.append(f)
861
862         return (finalfiles, masked)
863
864     def parse_bbfiles(self, filelist, masked):
865         parsed, cached, skipped, error, total = 0, 0, 0, 0, len(filelist)
866         for i in xrange(total):
867             f = filelist[i]
868
869             #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
870
871             # read a file's metadata
872             try:
873                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data, self.status)
874                 if skip:
875                     skipped += 1
876                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
877                     self.bb_cache.skip(f)
878                     continue
879                 elif fromCache: cached += 1
880                 else: parsed += 1
881
882                 # Disabled by RP as was no longer functional
883                 # allow metadata files to add items to BBFILES
884                 #data.update_data(self.pkgdata[f])
885                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
886                 #if addbbfiles:
887                 #    for aof in addbbfiles.split():
888                 #        if not files.count(aof):
889                 #            if not os.path.isabs(aof):
890                 #                aof = os.path.join(os.path.dirname(f),aof)
891                 #            files.append(aof)
892
893             except IOError, e:
894                 error += 1
895                 self.bb_cache.remove(f)
896                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
897                 pass
898             except KeyboardInterrupt:
899                 self.bb_cache.sync()
900                 raise
901             except Exception, e:
902                 error += 1
903                 self.bb_cache.remove(f)
904                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
905             except:
906                 self.bb_cache.remove(f)
907                 raise
908             finally:
909                 bb.event.fire(bb.event.ParseProgress(self.configuration.event_data, cached, parsed, skipped, masked, error, total))
910
911         self.bb_cache.sync()
912         if error > 0:
913              raise ParsingErrorsFound
914
915     def serve(self):
916
917         # Empty the environment. The environment will be populated as
918         # necessary from the data store.
919         bb.utils.empty_environment()
920
921         if self.configuration.profile:
922             try:
923                 import cProfile as profile
924             except:
925                 import profile
926
927             profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
928
929             # Redirect stdout to capture profile information
930             pout = open('profile.log.processed', 'w')
931             so = sys.stdout.fileno()
932             os.dup2(pout.fileno(), so)
933
934             import pstats
935             p = pstats.Stats('profile.log')
936             p.sort_stats('time')
937             p.print_stats()
938             p.print_callers()
939             p.sort_stats('cumulative')
940             p.print_stats()
941
942             os.dup2(so, pout.fileno())
943             pout.flush()
944             pout.close()
945         else:
946             self.server.serve_forever()
947         
948         bb.event.fire(CookerExit(self.configuration.event_data))
949         
950 class CookerExit(bb.event.Event):
951     """
952     Notify clients of the Cooker shutdown
953     """
954
955     def __init__(self, d):
956         bb.event.Event.__init__(self, d)
957