Fix GraphViz .dot output for rdepends and rrecs
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006 - 2007 Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from bb import command
29 import itertools, sre_constants
30
31 class MultipleMatches(Exception):
32     """
33     Exception raised when multiple file matches are found
34     """
35
36 class ParsingErrorsFound(Exception):
37     """
38     Exception raised when parsing errors are found
39     """
40
41 class NothingToBuild(Exception):
42     """
43     Exception raised when there is nothing to build
44     """
45
46
47 # Different states cooker can be in
48 cookerClean = 1
49 cookerParsing = 2
50 cookerParsed = 3
51
52 # Different action states the cooker can be in
53 cookerRun = 1           # Cooker is running normally
54 cookerShutdown = 2      # Active tasks should be brought to a controlled stop
55 cookerStop = 3          # Stop, now!
56
57 #============================================================================#
58 # BBCooker
59 #============================================================================#
60 class BBCooker:
61     """
62     Manages one bitbake build run
63     """
64
65     def __init__(self, configuration, server):
66         self.status = None
67
68         self.cache = None
69         self.bb_cache = None
70
71         self.server = server.BitBakeServer(self)
72
73         self.configuration = configuration
74
75         if self.configuration.verbose:
76             bb.msg.set_verbose(True)
77
78         if self.configuration.debug:
79             bb.msg.set_debug_level(self.configuration.debug)
80         else:
81             bb.msg.set_debug_level(0)
82
83         if self.configuration.debug_domains:
84             bb.msg.set_debug_domains(self.configuration.debug_domains)
85
86         self.configuration.data = bb.data.init()
87
88         bb.data.inheritFromOS(self.configuration.data)
89
90         self.parseConfigurationFiles(self.configuration.file)
91
92         if not self.configuration.cmd:
93             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
94
95         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
96         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
97             self.configuration.pkgs_to_build.extend(bbpkgs.split())
98
99         #
100         # Special updated configuration we use for firing events
101         #
102         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
103         bb.data.update_data(self.configuration.event_data)
104
105         # TOSTOP must not be set or our children will hang when they output
106         fd = sys.stdout.fileno()
107         if os.isatty(fd):
108             import termios
109             tcattr = termios.tcgetattr(fd)
110             if tcattr[3] & termios.TOSTOP:
111                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
112                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
113                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
114
115         self.command = bb.command.Command(self)
116         self.cookerState = cookerClean
117         self.cookerAction = cookerRun
118
119     def parseConfiguration(self):
120
121
122         # Change nice level if we're asked to
123         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
124         if nice:
125             curnice = os.nice(0)
126             nice = int(nice) - curnice
127             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
128
129     def parseCommandLine(self):
130         # Parse any commandline into actions
131         if self.configuration.show_environment:
132             self.commandlineAction = None
133
134             if 'world' in self.configuration.pkgs_to_build:
135                 bb.error("'world' is not a valid target for --environment.")
136             elif len(self.configuration.pkgs_to_build) > 1:
137                 bb.error("Only one target can be used with the --environment option.")
138             elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
139                 bb.error("No target should be used with the --environment and --buildfile options.")
140             elif len(self.configuration.pkgs_to_build) > 0:
141                 self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
142             else:
143                 self.commandlineAction = ["showEnvironment", self.configuration.buildfile]
144         elif self.configuration.buildfile is not None:
145             self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
146         elif self.configuration.revisions_changed:
147             self.commandlineAction = ["compareRevisions"]
148         elif self.configuration.show_versions:
149             self.commandlineAction = ["showVersions"]
150         elif self.configuration.parse_only:
151             self.commandlineAction = ["parseFiles"]
152         # FIXME - implement
153         #elif self.configuration.interactive:
154         #    self.interactiveMode()
155         elif self.configuration.dot_graph:
156             if self.configuration.pkgs_to_build:
157                 self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
158             else:
159                 self.commandlineAction = None
160                 bb.error("Please specify a package name for dependency graph generation.")
161         else:
162             if self.configuration.pkgs_to_build:
163                 self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
164             else:
165                 self.commandlineAction = None
166                 bb.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
167
168     def runCommands(self, server, data, abort):
169         """
170         Run any queued asynchronous command
171         This is done by the idle handler so it runs in true context rather than
172         tied to any UI.
173         """
174
175         return self.command.runAsyncCommand()
176
177     def tryBuildPackage(self, fn, item, task, the_data):
178         """
179         Build one task of a package, optionally build following task depends
180         """
181         try:
182             if not self.configuration.dry_run:
183                 bb.build.exec_task('do_%s' % task, the_data)
184             return True
185         except bb.build.FuncFailed:
186             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
187             raise
188         except bb.build.EventException, e:
189             event = e.args[1]
190             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
191             raise
192
193     def tryBuild(self, fn, task):
194         """
195         Build a provider and its dependencies. 
196         build_depends is a list of previous build dependencies (not runtime)
197         If build_depends is empty, we're dealing with a runtime depends
198         """
199
200         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
201
202         item = self.status.pkg_fn[fn]
203
204         #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
205         #    return True
206
207         return self.tryBuildPackage(fn, item, task, the_data)
208
209     def showVersions(self):
210
211         # Need files parsed
212         self.updateCache()
213
214         pkg_pn = self.status.pkg_pn
215         preferred_versions = {}
216         latest_versions = {}
217
218         # Sort by priority
219         for pn in pkg_pn:
220             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
221             preferred_versions[pn] = (pref_ver, pref_file)
222             latest_versions[pn] = (last_ver, last_file)
223
224         bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version"))
225         bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "================="))
226
227         for p in sorted(pkg_pn):
228             pref = preferred_versions[p]
229             latest = latest_versions[p]
230
231             prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
232             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
233
234             if pref == latest:
235                 prefstr = ""
236
237             bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr))
238
239     def compareRevisions(self):
240         ret = bb.fetch.fetcher_compare_revisons(self.configuration.data)
241         bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data)
242
243     def showEnvironment(self, buildfile = None, pkgs_to_build = []):
244         """
245         Show the outer or per-package environment
246         """
247         fn = None
248         envdata = None
249
250         if buildfile:
251             self.cb = None
252             self.bb_cache = bb.cache.init(self)
253             fn = self.matchFile(buildfile)
254         elif len(pkgs_to_build) == 1:
255             self.updateCache()
256
257             localdata = data.createCopy(self.configuration.data)
258             bb.data.update_data(localdata)
259             bb.data.expandKeys(localdata)
260
261             taskdata = bb.taskdata.TaskData(self.configuration.abort)
262             taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
263             taskdata.add_unresolved(localdata, self.status)
264
265             targetid = taskdata.getbuild_id(pkgs_to_build[0])
266             fnid = taskdata.build_targets[targetid][0]
267             fn = taskdata.fn_index[fnid]
268         else:
269             envdata = self.configuration.data
270
271         if fn:
272             try:
273                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
274             except IOError, e:
275                 bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
276                 raise
277             except Exception, e:
278                 bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
279                 raise
280
281         class dummywrite:
282             def __init__(self):
283                 self.writebuf = ""
284             def write(self, output):
285                 self.writebuf = self.writebuf + output
286
287         # emit variables and shell functions
288         try:
289             data.update_data(envdata)
290             wb = dummywrite()
291             data.emit_env(wb, envdata, True)
292             bb.msg.plain(wb.writebuf)
293         except Exception, e:
294             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
295         # emit the metadata which isnt valid shell
296         data.expandKeys(envdata)
297         for e in envdata.keys():
298             if data.getVarFlag( e, 'python', envdata ):
299                 bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
300
301     def generateDepTreeData(self, pkgs_to_build, task):
302         """
303         Create a dependency tree of pkgs_to_build, returning the data.
304         """
305
306         # Need files parsed
307         self.updateCache()
308
309         # If we are told to do the None task then query the default task
310         if (task == None):
311             task = self.configuration.cmd
312
313         pkgs_to_build = self.checkPackages(pkgs_to_build)
314
315         localdata = data.createCopy(self.configuration.data)
316         bb.data.update_data(localdata)
317         bb.data.expandKeys(localdata)
318         taskdata = bb.taskdata.TaskData(self.configuration.abort)
319
320         runlist = []
321         for k in pkgs_to_build:
322             taskdata.add_provider(localdata, self.status, k)
323             runlist.append([k, "do_%s" % task])
324         taskdata.add_unresolved(localdata, self.status)
325
326         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
327         rq.prepare_runqueue()
328
329         seen_fnids = []  
330         depend_tree = {}
331         depend_tree["depends"] = {}
332         depend_tree["tdepends"] = {}
333         depend_tree["pn"] = {}
334         depend_tree["rdepends-pn"] = {}
335         depend_tree["packages"] = {}
336         depend_tree["rdepends-pkg"] = {}
337         depend_tree["rrecs-pkg"] = {}
338
339         for task in range(len(rq.runq_fnid)):
340             taskname = rq.runq_task[task]
341             fnid = rq.runq_fnid[task]
342             fn = taskdata.fn_index[fnid]
343             pn = self.status.pkg_fn[fn]
344             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
345             if pn not in depend_tree["pn"]:
346                 depend_tree["pn"][pn] = {}
347                 depend_tree["pn"][pn]["filename"] = fn
348                 depend_tree["pn"][pn]["version"] = version
349             for dep in rq.runq_depends[task]:
350                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
351                 deppn = self.status.pkg_fn[depfn]
352                 dotname = "%s.%s" % (pn, rq.runq_task[task])
353                 if not dotname in depend_tree["tdepends"]:
354                     depend_tree["tdepends"][dotname] = []
355                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep]))
356             if fnid not in seen_fnids:
357                 seen_fnids.append(fnid)
358                 packages = []
359
360                 depend_tree["depends"][pn] = []
361                 for dep in taskdata.depids[fnid]:
362                     depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
363
364                 depend_tree["rdepends-pn"][pn] = []
365                 for rdep in taskdata.rdepids[fnid]:
366                         depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
367
368                 rdepends = self.status.rundeps[fn]
369                 for package in rdepends:
370                     depend_tree["rdepends-pkg"][package] = []
371                     for rdepend in bb.utils.explode_deps(rdepends[package]):
372                         depend_tree["rdepends-pkg"][package].append(rdepend)
373                     packages.append(package)
374
375                 rrecs = self.status.runrecs[fn]
376                 for package in rrecs:
377                     depend_tree["rrecs-pkg"][package] = []
378                     for rdepend in bb.utils.explode_deps(rrecs[package]):
379                         depend_tree["rrecs-pkg"][package].append(rdepend)
380                     if not package in packages:
381                         packages.append(package)
382
383                 for package in packages:
384                     if package not in depend_tree["packages"]:
385                         depend_tree["packages"][package] = {}
386                         depend_tree["packages"][package]["pn"] = pn
387                         depend_tree["packages"][package]["filename"] = fn
388                         depend_tree["packages"][package]["version"] = version
389
390         return depend_tree
391
392
393     def generateDepTreeEvent(self, pkgs_to_build, task):
394         """
395         Create a task dependency graph of pkgs_to_build.
396         Generate an event with the result
397         """
398         depgraph = self.generateDepTreeData(pkgs_to_build, task)
399         bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
400
401     def generateDotGraphFiles(self, pkgs_to_build, task):
402         """
403         Create a task dependency graph of pkgs_to_build.
404         Save the result to a set of .dot files.
405         """
406
407         depgraph = self.generateDepTreeData(pkgs_to_build, task)
408
409         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
410         depends_file = file('pn-depends.dot', 'w' )
411         print >> depends_file, "digraph depends {"
412         for pn in depgraph["pn"]:
413             fn = depgraph["pn"][pn]["filename"]
414             version = depgraph["pn"][pn]["version"]
415             print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
416         for pn in depgraph["depends"]:
417             for depend in depgraph["depends"][pn]:
418                 print >> depends_file, '"%s" -> "%s"' % (pn, depend)
419         for pn in depgraph["rdepends-pn"]:
420             for rdepend in depgraph["rdepends-pn"][pn]:
421                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend)
422         print >> depends_file,  "}"
423         bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
424
425         depends_file = file('package-depends.dot', 'w' )
426         print >> depends_file, "digraph depends {"
427         for package in depgraph["packages"]:
428             pn = depgraph["packages"][package]["pn"]
429             fn = depgraph["packages"][package]["filename"]
430             version = depgraph["packages"][package]["version"]
431             if package == pn:
432                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
433             else:
434                 print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
435             for depend in depgraph["depends"][pn]:
436                 print >> depends_file, '"%s" -> "%s"' % (package, depend)
437         for package in depgraph["rdepends-pkg"]:
438             for rdepend in depgraph["rdepends-pkg"][package]:
439                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
440         for package in depgraph["rrecs-pkg"]:
441             for rdepend in depgraph["rrecs-pkg"][package]:
442                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
443         print >> depends_file,  "}"
444         bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
445
446         tdepends_file = file('task-depends.dot', 'w' )
447         print >> tdepends_file, "digraph depends {"
448         for task in depgraph["tdepends"]:
449             (pn, taskname) = task.rsplit(".", 1)
450             fn = depgraph["pn"][pn]["filename"]
451             version = depgraph["pn"][pn]["version"]
452             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
453             for dep in depgraph["tdepends"][task]:
454                 print >> tdepends_file, '"%s" -> "%s"' % (task, dep)
455         print >> tdepends_file,  "}"
456         bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
457
458     def buildDepgraph( self ):
459         all_depends = self.status.all_depends
460         pn_provides = self.status.pn_provides
461
462         localdata = data.createCopy(self.configuration.data)
463         bb.data.update_data(localdata)
464         bb.data.expandKeys(localdata)
465
466         def calc_bbfile_priority(filename):
467             for (regex, pri) in self.status.bbfile_config_priorities:
468                 if regex.match(filename):
469                     return pri
470             return 0
471
472         # Handle PREFERRED_PROVIDERS
473         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
474             try:
475                 (providee, provider) = p.split(':')
476             except:
477                 bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
478                 continue
479             if providee in self.status.preferred and self.status.preferred[providee] != provider:
480                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
481             self.status.preferred[providee] = provider
482
483         # Calculate priorities for each file
484         for p in self.status.pkg_fn:
485             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
486
487     def buildWorldTargetList(self):
488         """
489          Build package list for "bitbake world"
490         """
491         all_depends = self.status.all_depends
492         pn_provides = self.status.pn_provides
493         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
494         for f in self.status.possible_world:
495             terminal = True
496             pn = self.status.pkg_fn[f]
497
498             for p in pn_provides[pn]:
499                 if p.startswith('virtual/'):
500                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
501                     terminal = False
502                     break
503                 for pf in self.status.providers[p]:
504                     if self.status.pkg_fn[pf] != pn:
505                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
506                         terminal = False
507                         break
508             if terminal:
509                 self.status.world_target.add(pn)
510
511             # drop reference count now
512             self.status.possible_world = None
513             self.status.all_depends    = None
514
515     def interactiveMode( self ):
516         """Drop off into a shell"""
517         try:
518             from bb import shell
519         except ImportError, details:
520             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
521         else:
522             shell.start( self )
523
524     def _findLayerConf(self):
525         path = os.getcwd()
526         while path != "/":
527             bblayers = os.path.join(path, "conf", "bblayers.conf")
528             if os.path.exists(bblayers):
529                 return bblayers
530
531             path, _ = os.path.split(path)
532
533     def parseConfigurationFiles(self, files):
534         try:
535             data = self.configuration.data
536             for f in files:
537                 data = bb.parse.handle(f, data)
538
539             layerconf = self._findLayerConf()
540             if layerconf:
541                 bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf)
542                 data = bb.parse.handle(layerconf, data)
543
544                 layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
545
546                 data = bb.data.createCopy(data)
547                 for layer in layers:
548                     bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer)
549                     bb.data.setVar('LAYERDIR', layer, data)
550                     data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data)
551
552                     # XXX: Hack, relies on the local keys of the datasmart
553                     # instance being stored in the 'dict' attribute and makes
554                     # assumptions about how variable expansion works, but
555                     # there's no better way to force an expansion of a single
556                     # variable across the datastore today, and this at least
557                     # lets us reference LAYERDIR without having to immediately
558                     # eval all our variables that use it.
559                     for key in data.dict:
560                         if key != "_data":
561                             value = data.getVar(key, False)
562                             if value and "${LAYERDIR}" in value:
563                                 data.setVar(key, value.replace("${LAYERDIR}", layer))
564
565                 bb.data.delVar('LAYERDIR', data)
566
567             if not data.getVar("BBPATH", True):
568                 bb.fatal("The BBPATH variable is not set")
569
570             data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data)
571
572             self.configuration.data = data
573
574             # Handle any INHERITs and inherit the base class
575             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
576             for inherit in inherits:
577                 self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
578
579             # Nomally we only register event handlers at the end of parsing .bb files
580             # We register any handlers we've found so far here...
581             for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
582                 bb.event.register(var,bb.data.getVar(var, self.configuration.data))
583
584             bb.fetch.fetcher_init(self.configuration.data)
585
586             bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
587
588
589         except IOError, e:
590             bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
591         except bb.parse.ParseError, details:
592             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) )
593
594     def handleCollections( self, collections ):
595         """Handle collections"""
596         if collections:
597             collection_list = collections.split()
598             for c in collection_list:
599                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
600                 if regex == None:
601                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
602                     continue
603                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
604                 if priority == None:
605                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
606                     continue
607                 try:
608                     cre = re.compile(regex)
609                 except re.error:
610                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
611                     continue
612                 try:
613                     pri = int(priority)
614                     self.status.bbfile_config_priorities.append((cre, pri))
615                 except ValueError:
616                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
617
618     def buildSetVars(self):
619         """
620         Setup any variables needed before starting a build
621         """
622         if not bb.data.getVar("BUILDNAME", self.configuration.data):
623             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
624         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data)
625
626     def matchFiles(self, buildfile):
627         """
628         Find the .bb files which match the expression in 'buildfile'.
629         """
630
631         bf = os.path.abspath(buildfile)
632         try:
633             os.stat(bf)
634             return [bf]
635         except OSError:
636             (filelist, masked) = self.collect_bbfiles()
637             regexp = re.compile(buildfile)
638             matches = []
639             for f in filelist:
640                 if regexp.search(f) and os.path.isfile(f):
641                     bf = f
642                     matches.append(f)
643             return matches
644
645     def matchFile(self, buildfile):
646         """
647         Find the .bb file which matches the expression in 'buildfile'.
648         Raise an error if multiple files
649         """
650         matches = self.matchFiles(buildfile)
651         if len(matches) != 1:
652             bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
653             for f in matches:
654                 bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
655             raise MultipleMatches
656         return matches[0]
657
658     def buildFile(self, buildfile, task):
659         """
660         Build the file matching regexp buildfile
661         """
662
663         # Parse the configuration here. We need to do it explicitly here since
664         # buildFile() doesn't use the cache
665         self.parseConfiguration()
666
667         # If we are told to do the None task then query the default task
668         if (task == None):
669             task = self.configuration.cmd
670
671         self.bb_cache = bb.cache.init(self)
672         self.status = bb.cache.CacheData()
673
674         (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile)
675         buildfile = self.matchFile(fn)
676         fn = self.bb_cache.realfn2virtual(buildfile, cls)
677
678         self.buildSetVars()
679
680         # Load data into the cache for fn and parse the loaded cache data
681         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
682         self.bb_cache.setData(fn, buildfile, the_data)
683         self.bb_cache.handle_data(fn, self.status)
684
685         # Tweak some variables
686         item = self.bb_cache.getVar('PN', fn, True)
687         self.status.ignored_dependencies = set()
688         self.status.bbfile_priority[fn] = 1
689
690         # Remove external dependencies
691         self.status.task_deps[fn]['depends'] = {}
692         self.status.deps[fn] = []
693         self.status.rundeps[fn] = []
694         self.status.runrecs[fn] = []
695
696         # Remove stamp for target if force mode active
697         if self.configuration.force:
698             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn))
699             bb.build.del_stamp('do_%s' % task, self.status, fn)
700
701         # Setup taskdata structure
702         taskdata = bb.taskdata.TaskData(self.configuration.abort)
703         taskdata.add_provider(self.configuration.data, self.status, item)
704
705         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
706         bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data)
707
708         # Execute the runqueue
709         runlist = [[item, "do_%s" % task]]
710
711         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
712
713         def buildFileIdle(server, rq, abort):
714
715             if abort or self.cookerAction == cookerStop:
716                 rq.finish_runqueue(True)
717             elif self.cookerAction == cookerShutdown:
718                 rq.finish_runqueue(False)
719             failures = 0
720             try:
721                 retval = rq.execute_runqueue()
722             except runqueue.TaskFailure, fnids:
723                 for fnid in fnids:
724                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
725                     failures = failures + 1
726                 retval = False
727             if not retval:
728                 bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
729                 self.command.finishAsyncCommand()
730                 return False
731             return 0.5
732
733         self.server.register_idle_function(buildFileIdle, rq)
734
735     def buildTargets(self, targets, task):
736         """
737         Attempt to build the targets specified
738         """
739
740         # Need files parsed
741         self.updateCache()
742
743         # If we are told to do the NULL task then query the default task
744         if (task == None):
745             task = self.configuration.cmd
746
747         targets = self.checkPackages(targets)
748
749         def buildTargetsIdle(server, rq, abort):
750
751             if abort or self.cookerAction == cookerStop:
752                 rq.finish_runqueue(True)
753             elif self.cookerAction == cookerShutdown:
754                 rq.finish_runqueue(False)
755             failures = 0
756             try:
757                 retval = rq.execute_runqueue()
758             except runqueue.TaskFailure, fnids:
759                 for fnid in fnids:
760                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
761                     failures = failures + 1
762                 retval = False
763             if not retval:
764                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data)
765                 self.command.finishAsyncCommand()
766                 return None
767             return 0.5
768
769         self.buildSetVars()
770
771         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
772         bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data)
773
774         localdata = data.createCopy(self.configuration.data)
775         bb.data.update_data(localdata)
776         bb.data.expandKeys(localdata)
777
778         taskdata = bb.taskdata.TaskData(self.configuration.abort)
779
780         runlist = []
781         for k in targets:
782             taskdata.add_provider(localdata, self.status, k)
783             runlist.append([k, "do_%s" % task])
784         taskdata.add_unresolved(localdata, self.status)
785
786         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
787
788         self.server.register_idle_function(buildTargetsIdle, rq)
789
790     def updateCache(self):
791
792         if self.cookerState == cookerParsed:
793             return
794
795         if self.cookerState != cookerParsing:
796
797             self.parseConfiguration ()
798
799             # Import Psyco if available and not disabled
800             import platform
801             if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
802                 if not self.configuration.disable_psyco:
803                     try:
804                         import psyco
805                     except ImportError:
806                         bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
807                     else:
808                         psyco.bind( CookerParser.parse_next )
809                 else:
810                     bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
811
812             self.status = bb.cache.CacheData()
813
814             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
815             self.status.ignored_dependencies = set(ignore.split())
816     
817             for dep in self.configuration.extra_assume_provided:
818                 self.status.ignored_dependencies.add(dep)
819     
820             self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
821
822             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
823             (filelist, masked) = self.collect_bbfiles()
824             bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
825
826             self.parser = CookerParser(self, filelist, masked)
827             self.cookerState = cookerParsing
828
829         if not self.parser.parse_next():
830             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
831             self.buildDepgraph()
832             self.cookerState = cookerParsed
833             return None
834
835         return True
836
837     def checkPackages(self, pkgs_to_build):
838
839         if len(pkgs_to_build) == 0:
840             raise NothingToBuild
841
842         if 'world' in pkgs_to_build:
843             self.buildWorldTargetList()
844             pkgs_to_build.remove('world')
845             for t in self.status.world_target:
846                 pkgs_to_build.append(t)
847
848         return pkgs_to_build
849
850     def get_bbfiles( self, path = os.getcwd() ):
851         """Get list of default .bb files by reading out the current directory"""
852         contents = os.listdir(path)
853         bbfiles = []
854         for f in contents:
855             (root, ext) = os.path.splitext(f)
856             if ext == ".bb":
857                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
858         return bbfiles
859
860     def find_bbfiles( self, path ):
861         """Find all the .bb files in a directory"""
862         from os.path import join
863
864         found = []
865         for dir, dirs, files in os.walk(path):
866             for ignored in ('SCCS', 'CVS', '.svn'):
867                 if ignored in dirs:
868                     dirs.remove(ignored)
869             found += [join(dir,f) for f in files if f.endswith('.bb')]
870
871         return found
872
873     def collect_bbfiles( self ):
874         """Collect all available .bb build files"""
875         parsed, cached, skipped, masked = 0, 0, 0, 0
876         self.bb_cache = bb.cache.init(self)
877
878         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
879         data.setVar("BBFILES", " ".join(files), self.configuration.data)
880
881         if not len(files):
882             files = self.get_bbfiles()
883
884         if not len(files):
885             bb.msg.error(bb.msg.domain.Collection, "no recipe files to build, check your BBPATH and BBFILES?")
886             bb.event.fire(CookerExit(), self.configuration.event_data)
887
888         newfiles = set()
889         for f in files:
890             if os.path.isdir(f):
891                 dirfiles = self.find_bbfiles(f)
892                 if dirfiles:
893                     newfiles.update(dirfiles)
894                     continue
895             else:
896                 globbed = glob.glob(f)
897                 if not globbed and os.path.exists(f):
898                     globbed = [f]
899                 newfiles.update(globbed)
900
901         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
902
903         if not bbmask:
904             return (list(newfiles), 0)
905
906         try:
907             bbmask_compiled = re.compile(bbmask)
908         except sre_constants.error:
909             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
910
911         finalfiles = []
912         for f in newfiles:
913             if bbmask_compiled.search(f):
914                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
915                 masked += 1
916                 continue
917             finalfiles.append(f)
918
919         return (finalfiles, masked)
920
921     def serve(self):
922
923         # Empty the environment. The environment will be populated as
924         # necessary from the data store.
925         bb.utils.empty_environment()
926
927         if self.configuration.profile:
928             try:
929                 import cProfile as profile
930             except:
931                 import profile
932
933             profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
934
935             # Redirect stdout to capture profile information
936             pout = open('profile.log.processed', 'w')
937             so = sys.stdout.fileno()
938             os.dup2(pout.fileno(), so)
939
940             import pstats
941             p = pstats.Stats('profile.log')
942             p.sort_stats('time')
943             p.print_stats()
944             p.print_callers()
945             p.sort_stats('cumulative')
946             p.print_stats()
947
948             os.dup2(so, pout.fileno())
949             pout.flush()
950             pout.close()
951         else:
952             self.server.serve_forever()
953         
954         bb.event.fire(CookerExit(), self.configuration.event_data)
955         
956 class CookerExit(bb.event.Event):
957     """
958     Notify clients of the Cooker shutdown
959     """
960
961     def __init__(self):
962         bb.event.Event.__init__(self)
963
964 class CookerParser:
965     def __init__(self, cooker, filelist, masked):
966         # Internal data
967         self.filelist = filelist
968         self.cooker = cooker
969
970         # Accounting statistics
971         self.parsed = 0
972         self.cached = 0
973         self.error = 0
974         self.masked = masked
975         self.total = len(filelist)
976
977         self.skipped = 0
978         self.virtuals = 0
979
980         # Pointer to the next file to parse
981         self.pointer = 0
982
983     def parse_next(self):
984         if self.pointer < len(self.filelist):
985             f = self.filelist[self.pointer]
986             cooker = self.cooker
987
988             try:
989                 fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
990                 if fromCache:
991                     self.cached += 1
992                 else:
993                     self.parsed += 1
994
995                 self.skipped += skipped
996                 self.virtuals += virtuals
997
998             except IOError, e:
999                 self.error += 1
1000                 cooker.bb_cache.remove(f)
1001                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
1002                 pass
1003             except KeyboardInterrupt:
1004                 cooker.bb_cache.remove(f)
1005                 cooker.bb_cache.sync()
1006                 raise
1007             except Exception, e:
1008                 self.error += 1
1009                 cooker.bb_cache.remove(f)
1010                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
1011             except:
1012                 cooker.bb_cache.remove(f)
1013                 raise
1014             finally:
1015                 bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data)
1016
1017             self.pointer += 1
1018
1019         if self.pointer >= self.total:
1020             cooker.bb_cache.sync()
1021             if self.error > 0:
1022                 raise ParsingErrorsFound
1023             return False
1024         return True
1025