Until the interactive mode is fixed, kill it from the valid options, to avoid confusion
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006 - 2007 Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 from __future__ import print_function
26 import sys, os, glob, os.path, re, time
27 import sre_constants
28 from cStringIO import StringIO
29 from contextlib import closing
30 import bb
31 from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
32
33 class MultipleMatches(Exception):
34     """
35     Exception raised when multiple file matches are found
36     """
37
38 class ParsingErrorsFound(Exception):
39     """
40     Exception raised when parsing errors are found
41     """
42
43 class NothingToBuild(Exception):
44     """
45     Exception raised when there is nothing to build
46     """
47
48
49 # Different states cooker can be in
50 cookerClean = 1
51 cookerParsing = 2
52 cookerParsed = 3
53
54 # Different action states the cooker can be in
55 cookerRun = 1           # Cooker is running normally
56 cookerShutdown = 2      # Active tasks should be brought to a controlled stop
57 cookerStop = 3          # Stop, now!
58
59 #============================================================================#
60 # BBCooker
61 #============================================================================#
62 class BBCooker:
63     """
64     Manages one bitbake build run
65     """
66
67     def __init__(self, configuration, server):
68         self.status = None
69
70         self.cache = None
71         self.bb_cache = None
72
73         self.server = server.BitBakeServer(self)
74
75         self.configuration = configuration
76
77         if self.configuration.verbose:
78             bb.msg.set_verbose(True)
79
80         if self.configuration.debug:
81             bb.msg.set_debug_level(self.configuration.debug)
82         else:
83             bb.msg.set_debug_level(0)
84
85         if self.configuration.debug_domains:
86             bb.msg.set_debug_domains(self.configuration.debug_domains)
87
88         self.configuration.data = bb.data.init()
89
90         bb.data.inheritFromOS(self.configuration.data)
91
92         self.parseConfigurationFiles(self.configuration.file)
93
94         if not self.configuration.cmd:
95             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
96
97         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
98         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
99             self.configuration.pkgs_to_build.extend(bbpkgs.split())
100
101         #
102         # Special updated configuration we use for firing events
103         #
104         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
105         bb.data.update_data(self.configuration.event_data)
106
107         # TOSTOP must not be set or our children will hang when they output
108         fd = sys.stdout.fileno()
109         if os.isatty(fd):
110             import termios
111             tcattr = termios.tcgetattr(fd)
112             if tcattr[3] & termios.TOSTOP:
113                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
114                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
115                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
116
117         self.command = bb.command.Command(self)
118         self.cookerState = cookerClean
119         self.cookerAction = cookerRun
120
121     def parseConfiguration(self):
122
123
124         # Change nice level if we're asked to
125         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
126         if nice:
127             curnice = os.nice(0)
128             nice = int(nice) - curnice
129             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
130
131     def parseCommandLine(self):
132         # Parse any commandline into actions
133         if self.configuration.show_environment:
134             self.commandlineAction = None
135
136             if 'world' in self.configuration.pkgs_to_build:
137                 bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.")
138             elif len(self.configuration.pkgs_to_build) > 1:
139                 bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.")
140             elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
141                 bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.")
142             elif len(self.configuration.pkgs_to_build) > 0:
143                 self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
144             else:
145                 self.commandlineAction = ["showEnvironment", self.configuration.buildfile]
146         elif self.configuration.buildfile is not None:
147             self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
148         elif self.configuration.revisions_changed:
149             self.commandlineAction = ["compareRevisions"]
150         elif self.configuration.show_versions:
151             self.commandlineAction = ["showVersions"]
152         elif self.configuration.parse_only:
153             self.commandlineAction = ["parseFiles"]
154         elif self.configuration.dot_graph:
155             if self.configuration.pkgs_to_build:
156                 self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
157             else:
158                 self.commandlineAction = None
159                 bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.")
160         else:
161             if self.configuration.pkgs_to_build:
162                 self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
163             else:
164                 self.commandlineAction = None
165                 bb.msg.error(bb.msg.domain.Build, "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
166
167     def runCommands(self, server, data, abort):
168         """
169         Run any queued asynchronous command
170         This is done by the idle handler so it runs in true context rather than
171         tied to any UI.
172         """
173
174         return self.command.runAsyncCommand()
175
176     def tryBuildPackage(self, fn, item, task, the_data):
177         """
178         Build one task of a package, optionally build following task depends
179         """
180         try:
181             if not self.configuration.dry_run:
182                 bb.build.exec_task('do_%s' % task, the_data)
183             return True
184         except bb.build.FuncFailed:
185             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
186             raise
187         except bb.build.EventException, e:
188             event = e.args[1]
189             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
190             raise
191
192     def tryBuild(self, fn, task):
193         """
194         Build a provider and its dependencies.
195         build_depends is a list of previous build dependencies (not runtime)
196         If build_depends is empty, we're dealing with a runtime depends
197         """
198
199         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
200
201         item = self.status.pkg_fn[fn]
202
203         #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
204         #    return True
205
206         return self.tryBuildPackage(fn, item, task, the_data)
207
208     def showVersions(self):
209
210         # Need files parsed
211         self.updateCache()
212
213         pkg_pn = self.status.pkg_pn
214         preferred_versions = {}
215         latest_versions = {}
216
217         # Sort by priority
218         for pn in pkg_pn:
219             (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
220             preferred_versions[pn] = (pref_ver, pref_file)
221             latest_versions[pn] = (last_ver, last_file)
222
223         bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version"))
224         bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "================="))
225
226         for p in sorted(pkg_pn):
227             pref = preferred_versions[p]
228             latest = latest_versions[p]
229
230             prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
231             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
232
233             if pref == latest:
234                 prefstr = ""
235
236             bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr))
237
238     def compareRevisions(self):
239         ret = bb.fetch.fetcher_compare_revisons(self.configuration.data)
240         bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data)
241
242     def showEnvironment(self, buildfile = None, pkgs_to_build = []):
243         """
244         Show the outer or per-package environment
245         """
246         fn = None
247         envdata = None
248
249         if buildfile:
250             self.cb = None
251             self.bb_cache = bb.cache.init(self)
252             fn = self.matchFile(buildfile)
253         elif len(pkgs_to_build) == 1:
254             self.updateCache()
255
256             localdata = data.createCopy(self.configuration.data)
257             bb.data.update_data(localdata)
258             bb.data.expandKeys(localdata)
259
260             taskdata = bb.taskdata.TaskData(self.configuration.abort)
261             taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
262             taskdata.add_unresolved(localdata, self.status)
263
264             targetid = taskdata.getbuild_id(pkgs_to_build[0])
265             fnid = taskdata.build_targets[targetid][0]
266             fn = taskdata.fn_index[fnid]
267         else:
268             envdata = self.configuration.data
269
270         if fn:
271             try:
272                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
273             except IOError, e:
274                 bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
275                 raise
276             except Exception, e:
277                 bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
278                 raise
279
280         # emit variables and shell functions
281         try:
282             data.update_data(envdata)
283             with closing(StringIO()) as env:
284                 data.emit_env(env, envdata, True)
285                 bb.msg.plain(env.getvalue())
286         except Exception, e:
287             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
288
289         # emit the metadata which isnt valid shell
290         data.expandKeys(envdata)
291         for e in envdata.keys():
292             if data.getVarFlag( e, 'python', envdata ):
293                 bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
294
295     def generateDepTreeData(self, pkgs_to_build, task):
296         """
297         Create a dependency tree of pkgs_to_build, returning the data.
298         """
299
300         # Need files parsed
301         self.updateCache()
302
303         # If we are told to do the None task then query the default task
304         if (task == None):
305             task = self.configuration.cmd
306
307         pkgs_to_build = self.checkPackages(pkgs_to_build)
308
309         localdata = data.createCopy(self.configuration.data)
310         bb.data.update_data(localdata)
311         bb.data.expandKeys(localdata)
312         taskdata = bb.taskdata.TaskData(self.configuration.abort)
313
314         runlist = []
315         for k in pkgs_to_build:
316             taskdata.add_provider(localdata, self.status, k)
317             runlist.append([k, "do_%s" % task])
318         taskdata.add_unresolved(localdata, self.status)
319
320         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
321         rq.prepare_runqueue()
322
323         seen_fnids = []
324         depend_tree = {}
325         depend_tree["depends"] = {}
326         depend_tree["tdepends"] = {}
327         depend_tree["pn"] = {}
328         depend_tree["rdepends-pn"] = {}
329         depend_tree["packages"] = {}
330         depend_tree["rdepends-pkg"] = {}
331         depend_tree["rrecs-pkg"] = {}
332
333         for task in range(len(rq.runq_fnid)):
334             taskname = rq.runq_task[task]
335             fnid = rq.runq_fnid[task]
336             fn = taskdata.fn_index[fnid]
337             pn = self.status.pkg_fn[fn]
338             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
339             if pn not in depend_tree["pn"]:
340                 depend_tree["pn"][pn] = {}
341                 depend_tree["pn"][pn]["filename"] = fn
342                 depend_tree["pn"][pn]["version"] = version
343             for dep in rq.runq_depends[task]:
344                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
345                 deppn = self.status.pkg_fn[depfn]
346                 dotname = "%s.%s" % (pn, rq.runq_task[task])
347                 if not dotname in depend_tree["tdepends"]:
348                     depend_tree["tdepends"][dotname] = []
349                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep]))
350             if fnid not in seen_fnids:
351                 seen_fnids.append(fnid)
352                 packages = []
353
354                 depend_tree["depends"][pn] = []
355                 for dep in taskdata.depids[fnid]:
356                     depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
357
358                 depend_tree["rdepends-pn"][pn] = []
359                 for rdep in taskdata.rdepids[fnid]:
360                     depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
361
362                 rdepends = self.status.rundeps[fn]
363                 for package in rdepends:
364                     depend_tree["rdepends-pkg"][package] = []
365                     for rdepend in rdepends[package]:
366                         depend_tree["rdepends-pkg"][package].append(rdepend)
367                     packages.append(package)
368
369                 rrecs = self.status.runrecs[fn]
370                 for package in rrecs:
371                     depend_tree["rrecs-pkg"][package] = []
372                     for rdepend in rrecs[package]:
373                         depend_tree["rrecs-pkg"][package].append(rdepend)
374                     if not package in packages:
375                         packages.append(package)
376
377                 for package in packages:
378                     if package not in depend_tree["packages"]:
379                         depend_tree["packages"][package] = {}
380                         depend_tree["packages"][package]["pn"] = pn
381                         depend_tree["packages"][package]["filename"] = fn
382                         depend_tree["packages"][package]["version"] = version
383
384         return depend_tree
385
386
387     def generateDepTreeEvent(self, pkgs_to_build, task):
388         """
389         Create a task dependency graph of pkgs_to_build.
390         Generate an event with the result
391         """
392         depgraph = self.generateDepTreeData(pkgs_to_build, task)
393         bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
394
395     def generateDotGraphFiles(self, pkgs_to_build, task):
396         """
397         Create a task dependency graph of pkgs_to_build.
398         Save the result to a set of .dot files.
399         """
400
401         depgraph = self.generateDepTreeData(pkgs_to_build, task)
402
403         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
404         depends_file = file('pn-depends.dot', 'w' )
405         print("digraph depends {", file=depends_file)
406         for pn in depgraph["pn"]:
407             fn = depgraph["pn"][pn]["filename"]
408             version = depgraph["pn"][pn]["version"]
409             print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
410         for pn in depgraph["depends"]:
411             for depend in depgraph["depends"][pn]:
412                 print('"%s" -> "%s"' % (pn, depend), file=depends_file)
413         for pn in depgraph["rdepends-pn"]:
414             for rdepend in depgraph["rdepends-pn"][pn]:
415                 print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
416         print("}", file=depends_file)
417         bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
418
419         depends_file = file('package-depends.dot', 'w' )
420         print("digraph depends {", file=depends_file)
421         for package in depgraph["packages"]:
422             pn = depgraph["packages"][package]["pn"]
423             fn = depgraph["packages"][package]["filename"]
424             version = depgraph["packages"][package]["version"]
425             if package == pn:
426                 print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
427             else:
428                 print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
429             for depend in depgraph["depends"][pn]:
430                 print('"%s" -> "%s"' % (package, depend), file=depends_file)
431         for package in depgraph["rdepends-pkg"]:
432             for rdepend in depgraph["rdepends-pkg"][package]:
433                 print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
434         for package in depgraph["rrecs-pkg"]:
435             for rdepend in depgraph["rrecs-pkg"][package]:
436                 print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
437         print("}", file=depends_file)
438         bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
439
440         tdepends_file = file('task-depends.dot', 'w' )
441         print("digraph depends {", file=tdepends_file)
442         for task in depgraph["tdepends"]:
443             (pn, taskname) = task.rsplit(".", 1)
444             fn = depgraph["pn"][pn]["filename"]
445             version = depgraph["pn"][pn]["version"]
446             print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
447             for dep in depgraph["tdepends"][task]:
448                 print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
449         print("}", file=tdepends_file)
450         bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
451
452     def buildDepgraph( self ):
453         all_depends = self.status.all_depends
454         pn_provides = self.status.pn_provides
455
456         localdata = data.createCopy(self.configuration.data)
457         bb.data.update_data(localdata)
458         bb.data.expandKeys(localdata)
459
460         def calc_bbfile_priority(filename):
461             for (regex, pri) in self.status.bbfile_config_priorities:
462                 if regex.match(filename):
463                     return pri
464             return 0
465
466         # Handle PREFERRED_PROVIDERS
467         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
468             try:
469                 (providee, provider) = p.split(':')
470             except:
471                 bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
472                 continue
473             if providee in self.status.preferred and self.status.preferred[providee] != provider:
474                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
475             self.status.preferred[providee] = provider
476
477         # Calculate priorities for each file
478         for p in self.status.pkg_fn:
479             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
480
481     def buildWorldTargetList(self):
482         """
483          Build package list for "bitbake world"
484         """
485         all_depends = self.status.all_depends
486         pn_provides = self.status.pn_provides
487         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
488         for f in self.status.possible_world:
489             terminal = True
490             pn = self.status.pkg_fn[f]
491
492             for p in pn_provides[pn]:
493                 if p.startswith('virtual/'):
494                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
495                     terminal = False
496                     break
497                 for pf in self.status.providers[p]:
498                     if self.status.pkg_fn[pf] != pn:
499                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
500                         terminal = False
501                         break
502             if terminal:
503                 self.status.world_target.add(pn)
504
505             # drop reference count now
506             self.status.possible_world = None
507             self.status.all_depends    = None
508
509     def interactiveMode( self ):
510         """Drop off into a shell"""
511         try:
512             from bb import shell
513         except ImportError, details:
514             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
515         else:
516             shell.start( self )
517
518     def _findLayerConf(self):
519         path = os.getcwd()
520         while path != "/":
521             bblayers = os.path.join(path, "conf", "bblayers.conf")
522             if os.path.exists(bblayers):
523                 return bblayers
524
525             path, _ = os.path.split(path)
526
527     def parseConfigurationFiles(self, files):
528         try:
529             data = self.configuration.data
530             for f in files:
531                 data = bb.parse.handle(f, data)
532
533             layerconf = self._findLayerConf()
534             if layerconf:
535                 bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf)
536                 data = bb.parse.handle(layerconf, data)
537
538                 layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
539
540                 data = bb.data.createCopy(data)
541                 for layer in layers:
542                     bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer)
543                     bb.data.setVar('LAYERDIR', layer, data)
544                     data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data)
545
546                     # XXX: Hack, relies on the local keys of the datasmart
547                     # instance being stored in the 'dict' attribute and makes
548                     # assumptions about how variable expansion works, but
549                     # there's no better way to force an expansion of a single
550                     # variable across the datastore today, and this at least
551                     # lets us reference LAYERDIR without having to immediately
552                     # eval all our variables that use it.
553                     for key in data.dict:
554                         if key != "_data":
555                             value = data.getVar(key, False)
556                             if "${LAYERDIR}" in value:
557                                 data.setVar(key, value.replace("${LAYERDIR}", layer))
558
559                 bb.data.delVar('LAYERDIR', data)
560
561             if not data.getVar("BBPATH", True):
562                 bb.fatal("The BBPATH variable is not set")
563
564             data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data)
565
566             self.configuration.data = data
567
568             # Handle any INHERITs and inherit the base class
569             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
570             for inherit in inherits:
571                 self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
572
573             # Nomally we only register event handlers at the end of parsing .bb files
574             # We register any handlers we've found so far here...
575             for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []:
576                 bb.event.register(var, bb.data.getVar(var, self.configuration.data))
577
578             bb.fetch.fetcher_init(self.configuration.data)
579
580             bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
581
582
583         except IOError, e:
584             bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
585         except bb.parse.ParseError, details:
586             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) )
587
588     def handleCollections( self, collections ):
589         """Handle collections"""
590         if collections:
591             collection_list = collections.split()
592             for c in collection_list:
593                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
594                 if regex == None:
595                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
596                     continue
597                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
598                 if priority == None:
599                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
600                     continue
601                 try:
602                     cre = re.compile(regex)
603                 except re.error:
604                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
605                     continue
606                 try:
607                     pri = int(priority)
608                     self.status.bbfile_config_priorities.append((cre, pri))
609                 except ValueError:
610                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
611
612     def buildSetVars(self):
613         """
614         Setup any variables needed before starting a build
615         """
616         if not bb.data.getVar("BUILDNAME", self.configuration.data):
617             bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data)
618         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data)
619
620     def matchFiles(self, buildfile):
621         """
622         Find the .bb files which match the expression in 'buildfile'.
623         """
624
625         bf = os.path.abspath(buildfile)
626         try:
627             os.stat(bf)
628             return [bf]
629         except OSError:
630             (filelist, masked) = self.collect_bbfiles()
631             regexp = re.compile(buildfile)
632             matches = []
633             for f in filelist:
634                 if regexp.search(f) and os.path.isfile(f):
635                     bf = f
636                     matches.append(f)
637             return matches
638
639     def matchFile(self, buildfile):
640         """
641         Find the .bb file which matches the expression in 'buildfile'.
642         Raise an error if multiple files
643         """
644         matches = self.matchFiles(buildfile)
645         if len(matches) != 1:
646             bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
647             for f in matches:
648                 bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
649             raise MultipleMatches
650         return matches[0]
651
652     def buildFile(self, buildfile, task):
653         """
654         Build the file matching regexp buildfile
655         """
656
657         # Parse the configuration here. We need to do it explicitly here since
658         # buildFile() doesn't use the cache
659         self.parseConfiguration()
660
661         # If we are told to do the None task then query the default task
662         if (task == None):
663             task = self.configuration.cmd
664
665         self.bb_cache = bb.cache.init(self)
666         self.status = bb.cache.CacheData()
667
668         (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile)
669         buildfile = self.matchFile(fn)
670         fn = self.bb_cache.realfn2virtual(buildfile, cls)
671
672         self.buildSetVars()
673
674         # Load data into the cache for fn and parse the loaded cache data
675         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
676         self.bb_cache.setData(fn, buildfile, the_data)
677         self.bb_cache.handle_data(fn, self.status)
678
679         # Tweak some variables
680         item = self.bb_cache.getVar('PN', fn, True)
681         self.status.ignored_dependencies = set()
682         self.status.bbfile_priority[fn] = 1
683
684         # Remove external dependencies
685         self.status.task_deps[fn]['depends'] = {}
686         self.status.deps[fn] = []
687         self.status.rundeps[fn] = []
688         self.status.runrecs[fn] = []
689
690         # Remove stamp for target if force mode active
691         if self.configuration.force:
692             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn))
693             bb.build.del_stamp('do_%s' % task, self.status, fn)
694
695         # Setup taskdata structure
696         taskdata = bb.taskdata.TaskData(self.configuration.abort)
697         taskdata.add_provider(self.configuration.data, self.status, item)
698
699         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
700         bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data)
701
702         # Execute the runqueue
703         runlist = [[item, "do_%s" % task]]
704
705         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
706
707         def buildFileIdle(server, rq, abort):
708
709             if abort or self.cookerAction == cookerStop:
710                 rq.finish_runqueue(True)
711             elif self.cookerAction == cookerShutdown:
712                 rq.finish_runqueue(False)
713             failures = 0
714             try:
715                 retval = rq.execute_runqueue()
716             except runqueue.TaskFailure as exc:
717                 for fnid in exc.args:
718                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
719                     failures = failures + 1
720                 retval = False
721             if not retval:
722                 bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
723                 self.command.finishAsyncCommand()
724                 return False
725             return 0.5
726
727         self.server.register_idle_function(buildFileIdle, rq)
728
729     def buildTargets(self, targets, task):
730         """
731         Attempt to build the targets specified
732         """
733
734         # Need files parsed
735         self.updateCache()
736
737         # If we are told to do the NULL task then query the default task
738         if (task == None):
739             task = self.configuration.cmd
740
741         targets = self.checkPackages(targets)
742
743         def buildTargetsIdle(server, rq, abort):
744
745             if abort or self.cookerAction == cookerStop:
746                 rq.finish_runqueue(True)
747             elif self.cookerAction == cookerShutdown:
748                 rq.finish_runqueue(False)
749             failures = 0
750             try:
751                 retval = rq.execute_runqueue()
752             except runqueue.TaskFailure as exc:
753                 for fnid in exc.args:
754                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
755                     failures = failures + 1
756                 retval = False
757             if not retval:
758                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data)
759                 self.command.finishAsyncCommand()
760                 return None
761             return 0.5
762
763         self.buildSetVars()
764
765         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
766         bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data)
767
768         localdata = data.createCopy(self.configuration.data)
769         bb.data.update_data(localdata)
770         bb.data.expandKeys(localdata)
771
772         taskdata = bb.taskdata.TaskData(self.configuration.abort)
773
774         runlist = []
775         for k in targets:
776             taskdata.add_provider(localdata, self.status, k)
777             runlist.append([k, "do_%s" % task])
778         taskdata.add_unresolved(localdata, self.status)
779
780         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
781
782         self.server.register_idle_function(buildTargetsIdle, rq)
783
784     def updateCache(self):
785
786         if self.cookerState == cookerParsed:
787             return
788
789         if self.cookerState != cookerParsing:
790
791             self.parseConfiguration ()
792
793             # Import Psyco if available and not disabled
794             import platform
795             if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
796                 if not self.configuration.disable_psyco:
797                     try:
798                         import psyco
799                     except ImportError:
800                         bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
801                     else:
802                         psyco.bind( CookerParser.parse_next )
803                 else:
804                     bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
805
806             self.status = bb.cache.CacheData()
807
808             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
809             self.status.ignored_dependencies = set(ignore.split())
810
811             for dep in self.configuration.extra_assume_provided:
812                 self.status.ignored_dependencies.add(dep)
813
814             self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
815
816             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
817             (filelist, masked) = self.collect_bbfiles()
818             bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
819
820             self.parser = CookerParser(self, filelist, masked)
821             self.cookerState = cookerParsing
822
823         if not self.parser.parse_next():
824             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
825             self.buildDepgraph()
826             self.cookerState = cookerParsed
827             return None
828
829         return True
830
831     def checkPackages(self, pkgs_to_build):
832
833         if len(pkgs_to_build) == 0:
834             raise NothingToBuild
835
836         if 'world' in pkgs_to_build:
837             self.buildWorldTargetList()
838             pkgs_to_build.remove('world')
839             for t in self.status.world_target:
840                 pkgs_to_build.append(t)
841
842         return pkgs_to_build
843
844     def get_bbfiles( self, path = os.getcwd() ):
845         """Get list of default .bb files by reading out the current directory"""
846         contents = os.listdir(path)
847         bbfiles = []
848         for f in contents:
849             (root, ext) = os.path.splitext(f)
850             if ext == ".bb":
851                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
852         return bbfiles
853
854     def find_bbfiles( self, path ):
855         """Find all the .bb files in a directory"""
856         from os.path import join
857
858         found = []
859         for dir, dirs, files in os.walk(path):
860             for ignored in ('SCCS', 'CVS', '.svn'):
861                 if ignored in dirs:
862                     dirs.remove(ignored)
863             found += [join(dir, f) for f in files if f.endswith('.bb')]
864
865         return found
866
867     def collect_bbfiles( self ):
868         """Collect all available .bb build files"""
869         parsed, cached, skipped, masked = 0, 0, 0, 0
870         self.bb_cache = bb.cache.init(self)
871
872         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
873         data.setVar("BBFILES", " ".join(files), self.configuration.data)
874
875         if not len(files):
876             files = self.get_bbfiles()
877
878         if not len(files):
879             bb.msg.error(bb.msg.domain.Collection, "no recipe files to build, check your BBPATH and BBFILES?")
880             bb.event.fire(CookerExit(), self.configuration.event_data)
881
882         newfiles = set()
883         for f in files:
884             if os.path.isdir(f):
885                 dirfiles = self.find_bbfiles(f)
886                 if dirfiles:
887                     newfiles.update(dirfiles)
888                     continue
889             else:
890                 globbed = glob.glob(f)
891                 if not globbed and os.path.exists(f):
892                     globbed = [f]
893                 newfiles.update(globbed)
894
895         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
896
897         if not bbmask:
898             return (list(newfiles), 0)
899
900         try:
901             bbmask_compiled = re.compile(bbmask)
902         except sre_constants.error:
903             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
904
905         finalfiles = []
906         for f in newfiles:
907             if bbmask_compiled.search(f):
908                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
909                 masked += 1
910                 continue
911             finalfiles.append(f)
912
913         return (finalfiles, masked)
914
915     def serve(self):
916
917         # Empty the environment. The environment will be populated as
918         # necessary from the data store.
919         bb.utils.empty_environment()
920
921         if self.configuration.profile:
922             try:
923                 import cProfile as profile
924             except:
925                 import profile
926
927             profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
928
929             # Redirect stdout to capture profile information
930             pout = open('profile.log.processed', 'w')
931             so = sys.stdout.fileno()
932             os.dup2(pout.fileno(), so)
933
934             import pstats
935             p = pstats.Stats('profile.log')
936             p.sort_stats('time')
937             p.print_stats()
938             p.print_callers()
939             p.sort_stats('cumulative')
940             p.print_stats()
941
942             os.dup2(so, pout.fileno())
943             pout.flush()
944             pout.close()
945         else:
946             self.server.serve_forever()
947
948         bb.event.fire(CookerExit(), self.configuration.event_data)
949
950 class CookerExit(bb.event.Event):
951     """
952     Notify clients of the Cooker shutdown
953     """
954
955     def __init__(self):
956         bb.event.Event.__init__(self)
957
958 class CookerParser:
959     def __init__(self, cooker, filelist, masked):
960         # Internal data
961         self.filelist = filelist
962         self.cooker = cooker
963
964         # Accounting statistics
965         self.parsed = 0
966         self.cached = 0
967         self.error = 0
968         self.masked = masked
969         self.total = len(filelist)
970
971         self.skipped = 0
972         self.virtuals = 0
973
974         # Pointer to the next file to parse
975         self.pointer = 0
976
977     def parse_next(self):
978         cooker = self.cooker
979         if self.pointer < len(self.filelist):
980             f = self.filelist[self.pointer]
981
982             try:
983                 fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
984                 if fromCache:
985                     self.cached += 1
986                 else:
987                     self.parsed += 1
988
989                 self.skipped += skipped
990                 self.virtuals += virtuals
991
992             except IOError, e:
993                 self.error += 1
994                 cooker.bb_cache.remove(f)
995                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
996                 pass
997             except KeyboardInterrupt:
998                 cooker.bb_cache.remove(f)
999                 cooker.bb_cache.sync()
1000                 raise
1001             except Exception, e:
1002                 self.error += 1
1003                 cooker.bb_cache.remove(f)
1004                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
1005             except:
1006                 cooker.bb_cache.remove(f)
1007                 raise
1008             finally:
1009                 bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data)
1010
1011             self.pointer += 1
1012
1013         if self.pointer >= self.total:
1014             cooker.bb_cache.sync()
1015             if self.error > 0:
1016                 raise ParsingErrorsFound
1017             return False
1018         return True