cooker: use logger.exception for config file parse errors
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006 - 2007 Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 from __future__ import print_function
26 import sys, os, glob, os.path, re, time
27 import atexit
28 import itertools
29 import logging
30 import multiprocessing
31 import sre_constants
32 import threading
33 from cStringIO import StringIO
34 from contextlib import closing
35 from functools import wraps
36 import bb, bb.exceptions
37 from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
38
39 logger      = logging.getLogger("BitBake")
40 collectlog  = logging.getLogger("BitBake.Collection")
41 buildlog    = logging.getLogger("BitBake.Build")
42 parselog    = logging.getLogger("BitBake.Parsing")
43 providerlog = logging.getLogger("BitBake.Provider")
44
45 class MultipleMatches(Exception):
46     """
47     Exception raised when multiple file matches are found
48     """
49
50 class NothingToBuild(Exception):
51     """
52     Exception raised when there is nothing to build
53     """
54
55 class state:
56     initial, parsing, running, shutdown, stop = range(5)
57
58 #============================================================================#
59 # BBCooker
60 #============================================================================#
61 class BBCooker:
62     """
63     Manages one bitbake build run
64     """
65
66     def __init__(self, configuration, server_registration_cb):
67         self.status = None
68         self.appendlist = {}
69
70         self.server_registration_cb = server_registration_cb
71
72         self.configuration = configuration
73
74         self.configuration.data = bb.data.init()
75
76         bb.data.inheritFromOS(self.configuration.data)
77
78         try:
79             self.parseConfigurationFiles(self.configuration.file)
80         except SyntaxError:
81             sys.exit(1)
82         except Exception:
83             logger.exception("Error parsing configuration files")
84             sys.exit(1)
85
86         if not self.configuration.cmd:
87             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
88
89         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
90         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
91             self.configuration.pkgs_to_build.extend(bbpkgs.split())
92
93         self.parseCommandLine()
94
95         #
96         # Special updated configuration we use for firing events
97         #
98         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
99         bb.data.update_data(self.configuration.event_data)
100
101         # TOSTOP must not be set or our children will hang when they output
102         fd = sys.stdout.fileno()
103         if os.isatty(fd):
104             import termios
105             tcattr = termios.tcgetattr(fd)
106             if tcattr[3] & termios.TOSTOP:
107                 buildlog.info("The terminal had the TOSTOP bit set, clearing...")
108                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
109                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
110
111         self.command = bb.command.Command(self)
112         self.state = state.initial
113
114         self.parser = None
115
116     def parseConfiguration(self):
117
118
119         # Change nice level if we're asked to
120         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
121         if nice:
122             curnice = os.nice(0)
123             nice = int(nice) - curnice
124             buildlog.verbose("Renice to %s " % os.nice(nice))
125
126     def parseCommandLine(self):
127         # Parse any commandline into actions
128         if self.configuration.show_environment:
129             self.commandlineAction = None
130
131             if 'world' in self.configuration.pkgs_to_build:
132                 buildlog.error("'world' is not a valid target for --environment.")
133             elif len(self.configuration.pkgs_to_build) > 1:
134                 buildlog.error("Only one target can be used with the --environment option.")
135             elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
136                 buildlog.error("No target should be used with the --environment and --buildfile options.")
137             elif len(self.configuration.pkgs_to_build) > 0:
138                 self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
139             else:
140                 self.commandlineAction = ["showEnvironment", self.configuration.buildfile]
141         elif self.configuration.buildfile is not None:
142             self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
143         elif self.configuration.revisions_changed:
144             self.commandlineAction = ["compareRevisions"]
145         elif self.configuration.show_versions:
146             self.commandlineAction = ["showVersions"]
147         elif self.configuration.parse_only:
148             self.commandlineAction = ["parseFiles"]
149         elif self.configuration.dot_graph:
150             if self.configuration.pkgs_to_build:
151                 self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
152             else:
153                 self.commandlineAction = None
154                 buildlog.error("Please specify a package name for dependency graph generation.")
155         else:
156             if self.configuration.pkgs_to_build:
157                 self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
158             else:
159                 self.commandlineAction = None
160                 buildlog.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
161
162     def runCommands(self, server, data, abort):
163         """
164         Run any queued asynchronous command
165         This is done by the idle handler so it runs in true context rather than
166         tied to any UI.
167         """
168
169         return self.command.runAsyncCommand()
170
171     def showVersions(self):
172
173         # Need files parsed
174         self.updateCache()
175
176         pkg_pn = self.status.pkg_pn
177         preferred_versions = {}
178         latest_versions = {}
179
180         # Sort by priority
181         for pn in pkg_pn:
182             (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
183             preferred_versions[pn] = (pref_ver, pref_file)
184             latest_versions[pn] = (last_ver, last_file)
185
186         logger.plain("%-35s %25s %25s", "Package Name", "Latest Version", "Preferred Version")
187         logger.plain("%-35s %25s %25s\n", "============", "==============", "=================")
188
189         for p in sorted(pkg_pn):
190             pref = preferred_versions[p]
191             latest = latest_versions[p]
192
193             prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
194             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
195
196             if pref == latest:
197                 prefstr = ""
198
199             logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
200
201     def showEnvironment(self, buildfile = None, pkgs_to_build = []):
202         """
203         Show the outer or per-package environment
204         """
205         fn = None
206         envdata = None
207
208         if buildfile:
209             fn = self.matchFile(buildfile)
210         elif len(pkgs_to_build) == 1:
211             self.updateCache()
212
213             localdata = data.createCopy(self.configuration.data)
214             bb.data.update_data(localdata)
215             bb.data.expandKeys(localdata)
216
217             taskdata = bb.taskdata.TaskData(self.configuration.abort)
218             taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
219             taskdata.add_unresolved(localdata, self.status)
220
221             targetid = taskdata.getbuild_id(pkgs_to_build[0])
222             fnid = taskdata.build_targets[targetid][0]
223             fn = taskdata.fn_index[fnid]
224         else:
225             envdata = self.configuration.data
226
227         if fn:
228             try:
229                 envdata = bb.cache.Cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
230             except Exception, e:
231                 parselog.exception("Unable to read %s", fn)
232                 raise
233
234         # emit variables and shell functions
235         data.update_data(envdata)
236         with closing(StringIO()) as env:
237             data.emit_env(env, envdata, True)
238             logger.plain(env.getvalue())
239
240         # emit the metadata which isnt valid shell
241         data.expandKeys(envdata)
242         for e in envdata.keys():
243             if data.getVarFlag( e, 'python', envdata ):
244                 logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
245
246     def prepareTreeData(self, pkgs_to_build, task):
247         """
248         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
249         """
250         # Need files parsed
251         self.updateCache()
252
253         # If we are told to do the None task then query the default task
254         if (task == None):
255             task = self.configuration.cmd
256
257         pkgs_to_build = self.checkPackages(pkgs_to_build)
258
259         localdata = data.createCopy(self.configuration.data)
260         bb.data.update_data(localdata)
261         bb.data.expandKeys(localdata)
262         taskdata = bb.taskdata.TaskData(self.configuration.abort)
263
264         runlist = []
265         for k in pkgs_to_build:
266             taskdata.add_provider(localdata, self.status, k)
267             runlist.append([k, "do_%s" % task])
268         taskdata.add_unresolved(localdata, self.status)
269
270         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
271         rq.rqdata.prepare()
272
273         return taskdata, rq
274
275     def generateDepTreeData(self, pkgs_to_build, task):
276         """
277         Create a dependency tree of pkgs_to_build, returning the data.
278         """
279         taskdata, rq = self.prepareTreeData(pkgs_to_build, task)
280
281         seen_fnids = []
282         depend_tree = {}
283         depend_tree["depends"] = {}
284         depend_tree["tdepends"] = {}
285         depend_tree["pn"] = {}
286         depend_tree["rdepends-pn"] = {}
287         depend_tree["packages"] = {}
288         depend_tree["rdepends-pkg"] = {}
289         depend_tree["rrecs-pkg"] = {}
290
291         for task in xrange(len(rq.rqdata.runq_fnid)):
292             taskname = rq.rqdata.runq_task[task]
293             fnid = rq.rqdata.runq_fnid[task]
294             fn = taskdata.fn_index[fnid]
295             pn = self.status.pkg_fn[fn]
296             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
297             if pn not in depend_tree["pn"]:
298                 depend_tree["pn"][pn] = {}
299                 depend_tree["pn"][pn]["filename"] = fn
300                 depend_tree["pn"][pn]["version"] = version
301             for dep in rq.rqdata.runq_depends[task]:
302                 depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
303                 deppn = self.status.pkg_fn[depfn]
304                 dotname = "%s.%s" % (pn, rq.rqdata.runq_task[task])
305                 if not dotname in depend_tree["tdepends"]:
306                     depend_tree["tdepends"][dotname] = []
307                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.rqdata.runq_task[dep]))
308             if fnid not in seen_fnids:
309                 seen_fnids.append(fnid)
310                 packages = []
311
312                 depend_tree["depends"][pn] = []
313                 for dep in taskdata.depids[fnid]:
314                     depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
315
316                 depend_tree["rdepends-pn"][pn] = []
317                 for rdep in taskdata.rdepids[fnid]:
318                     depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
319
320                 rdepends = self.status.rundeps[fn]
321                 for package in rdepends:
322                     depend_tree["rdepends-pkg"][package] = []
323                     for rdepend in rdepends[package]:
324                         depend_tree["rdepends-pkg"][package].append(rdepend)
325                     packages.append(package)
326
327                 rrecs = self.status.runrecs[fn]
328                 for package in rrecs:
329                     depend_tree["rrecs-pkg"][package] = []
330                     for rdepend in rrecs[package]:
331                         depend_tree["rrecs-pkg"][package].append(rdepend)
332                     if not package in packages:
333                         packages.append(package)
334
335                 for package in packages:
336                     if package not in depend_tree["packages"]:
337                         depend_tree["packages"][package] = {}
338                         depend_tree["packages"][package]["pn"] = pn
339                         depend_tree["packages"][package]["filename"] = fn
340                         depend_tree["packages"][package]["version"] = version
341
342         return depend_tree
343
344
345     def generateDepTreeEvent(self, pkgs_to_build, task):
346         """
347         Create a task dependency graph of pkgs_to_build.
348         Generate an event with the result
349         """
350         depgraph = self.generateDepTreeData(pkgs_to_build, task)
351         bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
352
353     def generateDotGraphFiles(self, pkgs_to_build, task):
354         """
355         Create a task dependency graph of pkgs_to_build.
356         Save the result to a set of .dot files.
357         """
358
359         depgraph = self.generateDepTreeData(pkgs_to_build, task)
360
361         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
362         depends_file = file('pn-depends.dot', 'w' )
363         print("digraph depends {", file=depends_file)
364         for pn in depgraph["pn"]:
365             fn = depgraph["pn"][pn]["filename"]
366             version = depgraph["pn"][pn]["version"]
367             print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
368         for pn in depgraph["depends"]:
369             for depend in depgraph["depends"][pn]:
370                 print('"%s" -> "%s"' % (pn, depend), file=depends_file)
371         for pn in depgraph["rdepends-pn"]:
372             for rdepend in depgraph["rdepends-pn"][pn]:
373                 print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
374         print("}", file=depends_file)
375         logger.info("PN dependencies saved to 'pn-depends.dot'")
376
377         depends_file = file('package-depends.dot', 'w' )
378         print("digraph depends {", file=depends_file)
379         for package in depgraph["packages"]:
380             pn = depgraph["packages"][package]["pn"]
381             fn = depgraph["packages"][package]["filename"]
382             version = depgraph["packages"][package]["version"]
383             if package == pn:
384                 print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
385             else:
386                 print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
387             for depend in depgraph["depends"][pn]:
388                 print('"%s" -> "%s"' % (package, depend), file=depends_file)
389         for package in depgraph["rdepends-pkg"]:
390             for rdepend in depgraph["rdepends-pkg"][package]:
391                 print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
392         for package in depgraph["rrecs-pkg"]:
393             for rdepend in depgraph["rrecs-pkg"][package]:
394                 print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
395         print("}", file=depends_file)
396         logger.info("Package dependencies saved to 'package-depends.dot'")
397
398         tdepends_file = file('task-depends.dot', 'w' )
399         print("digraph depends {", file=tdepends_file)
400         for task in depgraph["tdepends"]:
401             (pn, taskname) = task.rsplit(".", 1)
402             fn = depgraph["pn"][pn]["filename"]
403             version = depgraph["pn"][pn]["version"]
404             print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
405             for dep in depgraph["tdepends"][task]:
406                 print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
407         print("}", file=tdepends_file)
408         logger.info("Task dependencies saved to 'task-depends.dot'")
409
410     def buildDepgraph( self ):
411         all_depends = self.status.all_depends
412         pn_provides = self.status.pn_provides
413
414         localdata = data.createCopy(self.configuration.data)
415         bb.data.update_data(localdata)
416         bb.data.expandKeys(localdata)
417
418         matched = set()
419         def calc_bbfile_priority(filename):
420             for _, _, regex, pri in self.status.bbfile_config_priorities:
421                 if regex.match(filename):
422                     if not regex in matched:
423                         matched.add(regex)
424                     return pri
425             return 0
426
427         # Handle PREFERRED_PROVIDERS
428         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
429             try:
430                 (providee, provider) = p.split(':')
431             except:
432                 providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
433                 continue
434             if providee in self.status.preferred and self.status.preferred[providee] != provider:
435                 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee])
436             self.status.preferred[providee] = provider
437
438         # Calculate priorities for each file
439         for p in self.status.pkg_fn:
440             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
441
442         for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
443             if not regex in matched:
444                 collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
445
446     def findConfigFiles(self, varname):
447         """
448         Find config files which are appropriate values for varname.
449         i.e. MACHINE, DISTRO
450         """
451         possible = []
452         var = varname.lower()
453
454         data = self.configuration.data
455         # iterate configs
456         bbpaths = bb.data.getVar('BBPATH', data, True).split(':')
457         for path in bbpaths:
458             confpath = os.path.join(path, "conf", var)
459             if os.path.exists(confpath):
460                 for root, dirs, files in os.walk(confpath):
461                     # get all child files, these are appropriate values
462                     for f in files:
463                         val, sep, end = f.rpartition('.')
464                         if end == 'conf':
465                             possible.append(val)
466
467         bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.configuration.data)
468
469     def findInheritsClass(self, klass):
470         """
471         Find all recipes which inherit the specified class
472         """
473         pkg_list = []
474
475         for pfn in self.status.pkg_fn:
476             inherits = self.status.inherits.get(pfn, None)
477             if inherits and inherits.count(klass) > 0:
478                 pkg_list.append(self.status.pkg_fn[pfn])
479
480         return pkg_list
481
482     def generateTargetsTreeData(self, pkgs_to_build, task):
483         """
484         Create a tree of pkgs_to_build metadata, returning the data.
485         """
486         taskdata, rq = self.prepareTreeData(pkgs_to_build, task)
487
488         seen_fnids = []
489         target_tree = {}
490         target_tree["depends"] = {}
491         target_tree["pn"] = {}
492         target_tree["rdepends-pn"] = {}
493
494         for task in xrange(len(rq.rqdata.runq_fnid)):
495             taskname = rq.rqdata.runq_task[task]
496             fnid = rq.rqdata.runq_fnid[task]
497             fn = taskdata.fn_index[fnid]
498             pn = self.status.pkg_fn[fn]
499             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
500             summary = self.status.summary[fn]
501             license = self.status.license[fn]
502             section = self.status.section[fn]
503             if pn not in target_tree["pn"]:
504                 target_tree["pn"][pn] = {}
505                 target_tree["pn"][pn]["filename"] = fn
506                 target_tree["pn"][pn]["version"] = version
507                 target_tree["pn"][pn]["summary"] = summary
508                 target_tree["pn"][pn]["license"] = license
509                 target_tree["pn"][pn]["section"] = section
510             if fnid not in seen_fnids:
511                 seen_fnids.append(fnid)
512                 packages = []
513
514                 target_tree["depends"][pn] = []
515                 for dep in taskdata.depids[fnid]:
516                     target_tree["depends"][pn].append(taskdata.build_names_index[dep])
517
518                 target_tree["rdepends-pn"][pn] = []
519                 for rdep in taskdata.rdepids[fnid]:
520                     target_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
521
522         return target_tree
523
524     def generateTargetsTree(self, klass):
525         """
526         Generate a dependency tree of buildable targets
527         Generate an event with the result
528         """
529         pkgs = ['world']
530         # if inherited_class passed ensure all recipes which inherit the
531         # specified class are included in pkgs
532         if klass:
533             extra_pkgs = self.findInheritsClass(klass)
534             pkgs = pkgs + extra_pkgs
535
536         # generate a dependency tree for all our packages
537         tree = self.generateTargetsTreeData(pkgs, 'build')
538         bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
539
540     def buildWorldTargetList(self):
541         """
542          Build package list for "bitbake world"
543         """
544         all_depends = self.status.all_depends
545         pn_provides = self.status.pn_provides
546         parselog.debug(1, "collating packages for \"world\"")
547         for f in self.status.possible_world:
548             terminal = True
549             pn = self.status.pkg_fn[f]
550
551             for p in pn_provides[pn]:
552                 if p.startswith('virtual/'):
553                     parselog.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
554                     terminal = False
555                     break
556                 for pf in self.status.providers[p]:
557                     if self.status.pkg_fn[pf] != pn:
558                         parselog.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
559                         terminal = False
560                         break
561             if terminal:
562                 self.status.world_target.add(pn)
563
564     def interactiveMode( self ):
565         """Drop off into a shell"""
566         try:
567             from bb import shell
568         except ImportError:
569             parselog.exception("Interactive mode not available")
570             sys.exit(1)
571         else:
572             shell.start( self )
573
574     def _findLayerConf(self):
575         path = os.getcwd()
576         while path != "/":
577             bblayers = os.path.join(path, "conf", "bblayers.conf")
578             if os.path.exists(bblayers):
579                 return bblayers
580
581             path, _ = os.path.split(path)
582
583     def parseConfigurationFiles(self, files):
584         data = self.configuration.data
585         bb.parse.init_parser(data)
586         for f in files:
587             data = _parse(f, data)
588
589         layerconf = self._findLayerConf()
590         if layerconf:
591             parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
592             data = _parse(layerconf, data)
593
594             layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
595
596             data = bb.data.createCopy(data)
597             for layer in layers:
598                 parselog.debug(2, "Adding layer %s", layer)
599                 bb.data.setVar('LAYERDIR', layer, data)
600                 data = _parse(os.path.join(layer, "conf", "layer.conf"), data)
601                 data.expandVarref('LAYERDIR')
602
603             bb.data.delVar('LAYERDIR', data)
604
605         if not data.getVar("BBPATH", True):
606             raise SystemExit("The BBPATH variable is not set")
607
608         data = _parse(os.path.join("conf", "bitbake.conf"), data)
609
610         # Handle any INHERITs and inherit the base class
611         bbclasses  = ["base"] + (data.getVar('INHERIT', True) or "").split()
612         for bbclass in bbclasses:
613             data = _inherit(bbclass, data)
614
615         # Nomally we only register event handlers at the end of parsing .bb files
616         # We register any handlers we've found so far here...
617         for var in bb.data.getVar('__BBHANDLERS', data) or []:
618             bb.event.register(var, bb.data.getVar(var, data))
619
620         bb.fetch.fetcher_init(data)
621         bb.codeparser.parser_cache_init(data)
622         bb.parse.init_parser(data)
623         bb.event.fire(bb.event.ConfigParsed(), data)
624         self.configuration.data = data
625
626     def handleCollections( self, collections ):
627         """Handle collections"""
628         if collections:
629             collection_list = collections.split()
630             for c in collection_list:
631                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
632                 if regex == None:
633                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
634                     continue
635                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
636                 if priority == None:
637                     parselog.error("BBFILE_PRIORITY_%s not defined" % c)
638                     continue
639                 try:
640                     cre = re.compile(regex)
641                 except re.error:
642                     parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
643                     continue
644                 try:
645                     pri = int(priority)
646                     self.status.bbfile_config_priorities.append((c, regex, cre, pri))
647                 except ValueError:
648                     parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
649
650     def buildSetVars(self):
651         """
652         Setup any variables needed before starting a build
653         """
654         if not bb.data.getVar("BUILDNAME", self.configuration.data):
655             bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data)
656         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data)
657
658     def matchFiles(self, buildfile):
659         """
660         Find the .bb files which match the expression in 'buildfile'.
661         """
662
663         bf = os.path.abspath(buildfile)
664         filelist, masked = self.collect_bbfiles()
665         try:
666             os.stat(bf)
667             return [bf]
668         except OSError:
669             regexp = re.compile(buildfile)
670             matches = []
671             for f in filelist:
672                 if regexp.search(f) and os.path.isfile(f):
673                     bf = f
674                     matches.append(f)
675             return matches
676
677     def matchFile(self, buildfile):
678         """
679         Find the .bb file which matches the expression in 'buildfile'.
680         Raise an error if multiple files
681         """
682         matches = self.matchFiles(buildfile)
683         if len(matches) != 1:
684             parselog.error("Unable to match %s (%s matches found):" % (buildfile, len(matches)))
685             for f in matches:
686                 parselog.error("    %s" % f)
687             raise MultipleMatches
688         return matches[0]
689
690     def buildFile(self, buildfile, task):
691         """
692         Build the file matching regexp buildfile
693         """
694
695         # Parse the configuration here. We need to do it explicitly here since
696         # buildFile() doesn't use the cache
697         self.parseConfiguration()
698
699         # If we are told to do the None task then query the default task
700         if (task == None):
701             task = self.configuration.cmd
702
703         fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile)
704         fn = os.path.abspath(fn)
705         buildfile = self.matchFile(fn)
706
707         self.buildSetVars()
708
709         self.status = bb.cache.CacheData()
710         infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \
711                                      self.configuration.data)
712         infos = dict(infos)
713
714         fn = bb.cache.Cache.realfn2virtual(buildfile, cls)
715         try:
716             maininfo = infos[fn]
717         except KeyError:
718             bb.fatal("%s does not exist" % fn)
719         self.status.add_from_recipeinfo(fn, maininfo)
720
721         # Tweak some variables
722         item = maininfo.pn
723         self.status.ignored_dependencies = set()
724         self.status.bbfile_priority[fn] = 1
725
726         # Remove external dependencies
727         self.status.task_deps[fn]['depends'] = {}
728         self.status.deps[fn] = []
729         self.status.rundeps[fn] = []
730         self.status.runrecs[fn] = []
731
732         # Remove stamp for target if force mode active
733         if self.configuration.force:
734             logger.verbose("Remove stamp %s, %s", task, fn)
735             bb.build.del_stamp('do_%s' % task, self.status, fn)
736
737         # Setup taskdata structure
738         taskdata = bb.taskdata.TaskData(self.configuration.abort)
739         taskdata.add_provider(self.configuration.data, self.status, item)
740
741         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
742         bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data)
743
744         # Execute the runqueue
745         runlist = [[item, "do_%s" % task]]
746
747         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
748
749         def buildFileIdle(server, rq, abort):
750
751             if abort or self.state == state.stop:
752                 rq.finish_runqueue(True)
753             elif self.state == state.shutdown:
754                 rq.finish_runqueue(False)
755             failures = 0
756             try:
757                 retval = rq.execute_runqueue()
758             except runqueue.TaskFailure as exc:
759                 for fnid in exc.args:
760                     buildlog.error("'%s' failed" % taskdata.fn_index[fnid])
761                 failures += len(exc.args)
762                 retval = False
763             except SystemExit as exc:
764                 self.command.finishAsyncCommand()
765                 return False
766
767             if not retval:
768                 bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
769                 self.command.finishAsyncCommand()
770                 return False
771             if retval is True:
772                 return True
773             return 0.5
774
775         self.server_registration_cb(buildFileIdle, rq)
776
777     def buildTargets(self, targets, task):
778         """
779         Attempt to build the targets specified
780         """
781
782         # Need files parsed
783         self.updateCache()
784
785         # If we are told to do the NULL task then query the default task
786         if (task == None):
787             task = self.configuration.cmd
788
789         targets = self.checkPackages(targets)
790
791         def buildTargetsIdle(server, rq, abort):
792             if abort or self.state == state.stop:
793                 rq.finish_runqueue(True)
794             elif self.state == state.shutdown:
795                 rq.finish_runqueue(False)
796             failures = 0
797             try:
798                 retval = rq.execute_runqueue()
799             except runqueue.TaskFailure as exc:
800                 for fnid in exc.args:
801                     buildlog.error("'%s' failed" % taskdata.fn_index[fnid])
802                 failures += len(exc.args)
803                 retval = False
804             except SystemExit as exc:
805                 self.command.finishAsyncCommand()
806                 return False
807
808             if not retval:
809                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data)
810                 self.command.finishAsyncCommand()
811                 return False
812             if retval is True:
813                 return True
814             return 0.5
815
816         self.buildSetVars()
817
818         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
819         bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data)
820
821         localdata = data.createCopy(self.configuration.data)
822         bb.data.update_data(localdata)
823         bb.data.expandKeys(localdata)
824
825         taskdata = bb.taskdata.TaskData(self.configuration.abort)
826
827         runlist = []
828         for k in targets:
829             taskdata.add_provider(localdata, self.status, k)
830             runlist.append([k, "do_%s" % task])
831         taskdata.add_unresolved(localdata, self.status)
832
833         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
834
835         self.server_registration_cb(buildTargetsIdle, rq)
836
837     def updateCache(self):
838         if self.state == state.running:
839             return
840
841         if self.state in (state.shutdown, state.stop):
842             self.parser.shutdown(clean=False)
843             sys.exit(1)
844
845         if self.state != state.parsing:
846             self.parseConfiguration ()
847
848             # Import Psyco if available and not disabled
849             import platform
850             if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
851                 if not self.configuration.disable_psyco:
852                     try:
853                         import psyco
854                     except ImportError:
855                         collectlog.info("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
856                     else:
857                         psyco.bind( CookerParser.parse_next )
858                 else:
859                     collectlog.info("You have disabled Psyco. This decreases performance.")
860
861             self.status = bb.cache.CacheData()
862
863             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
864             self.status.ignored_dependencies = set(ignore.split())
865
866             for dep in self.configuration.extra_assume_provided:
867                 self.status.ignored_dependencies.add(dep)
868
869             self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
870
871             (filelist, masked) = self.collect_bbfiles()
872             bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
873
874             self.parser = CookerParser(self, filelist, masked)
875             self.state = state.parsing
876
877         if not self.parser.parse_next():
878             collectlog.debug(1, "parsing complete")
879             self.buildDepgraph()
880             self.state = state.running
881             return None
882
883         return True
884
885     def checkPackages(self, pkgs_to_build):
886
887         if len(pkgs_to_build) == 0:
888             raise NothingToBuild
889
890         if 'world' in pkgs_to_build:
891             self.buildWorldTargetList()
892             pkgs_to_build.remove('world')
893             for t in self.status.world_target:
894                 pkgs_to_build.append(t)
895
896         return pkgs_to_build
897
898     def get_bbfiles( self, path = os.getcwd() ):
899         """Get list of default .bb files by reading out the current directory"""
900         contents = os.listdir(path)
901         bbfiles = []
902         for f in contents:
903             (root, ext) = os.path.splitext(f)
904             if ext == ".bb":
905                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
906         return bbfiles
907
908     def find_bbfiles( self, path ):
909         """Find all the .bb and .bbappend files in a directory"""
910         from os.path import join
911
912         found = []
913         for dir, dirs, files in os.walk(path):
914             for ignored in ('SCCS', 'CVS', '.svn'):
915                 if ignored in dirs:
916                     dirs.remove(ignored)
917             found += [join(dir, f) for f in files if (f.endswith('.bb') or f.endswith('.bbappend'))]
918
919         return found
920
921     def collect_bbfiles( self ):
922         """Collect all available .bb build files"""
923         parsed, cached, skipped, masked = 0, 0, 0, 0
924
925         collectlog.debug(1, "collecting .bb files")
926
927         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
928         data.setVar("BBFILES", " ".join(files), self.configuration.data)
929
930         if not len(files):
931             files = self.get_bbfiles()
932
933         if not len(files):
934             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
935             bb.event.fire(CookerExit(), self.configuration.event_data)
936
937         newfiles = set()
938         for f in files:
939             if os.path.isdir(f):
940                 dirfiles = self.find_bbfiles(f)
941                 newfiles.update(dirfiles)
942             else:
943                 globbed = glob.glob(f)
944                 if not globbed and os.path.exists(f):
945                     globbed = [f]
946                 newfiles.update(globbed)
947
948         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
949
950         if bbmask:
951             try:
952                 bbmask_compiled = re.compile(bbmask)
953             except sre_constants.error:
954                 collectlog.critical("BBMASK is not a valid regular expression, ignoring.")
955                 return list(newfiles), 0
956
957         bbfiles = []
958         bbappend = []
959         for f in newfiles:
960             if bbmask and bbmask_compiled.search(f):
961                 collectlog.debug(1, "skipping masked file %s", f)
962                 masked += 1
963                 continue
964             if f.endswith('.bb'):
965                 bbfiles.append(f)
966             elif f.endswith('.bbappend'):
967                 bbappend.append(f)
968             else:
969                 collectlog.debug(1, "skipping %s: unknown file extension", f)
970
971         # Build a list of .bbappend files for each .bb file
972         for f in bbappend:
973             base = os.path.basename(f).replace('.bbappend', '.bb')
974             if not base in self.appendlist:
975                self.appendlist[base] = []
976             self.appendlist[base].append(f)
977
978         return (bbfiles, masked)
979
980     def get_file_appends(self, fn):
981         """
982         Returns a list of .bbappend files to apply to fn
983         NB: collect_bbfiles() must have been called prior to this
984         """
985         f = os.path.basename(fn)
986         if f in self.appendlist:
987             return self.appendlist[f]
988         return []
989
990     def shutdown(self):
991         self.state = state.shutdown
992
993     def stop(self):
994         self.state = state.stop
995
996 class CookerExit(bb.event.Event):
997     """
998     Notify clients of the Cooker shutdown
999     """
1000
1001     def __init__(self):
1002         bb.event.Event.__init__(self)
1003
1004 def catch_parse_error(func):
1005     """Exception handling bits for our parsing"""
1006     @wraps(func)
1007     def wrapped(fn, *args):
1008         try:
1009             return func(fn, *args)
1010         except (IOError, bb.parse.ParseError) as exc:
1011             parselog.critical("Unable to parse %s: %s" % (fn, exc))
1012             sys.exit(1)
1013     return wrapped
1014
1015 @catch_parse_error
1016 def _parse(fn, data, include=False):
1017     return bb.parse.handle(fn, data, include)
1018
1019 @catch_parse_error
1020 def _inherit(bbclass, data):
1021     bb.parse.BBHandler.inherit([bbclass], data)
1022     return data
1023
1024 class ParsingFailure(Exception):
1025     def __init__(self, realexception, recipe):
1026         self.realexception = realexception
1027         self.recipe = recipe
1028         Exception.__init__(self, realexception, recipe)
1029
1030 def parse_file(task):
1031     filename, appends = task
1032     try:
1033         return True, bb.cache.Cache.parse(filename, appends, parse_file.cfg)
1034     except Exception, exc:
1035         tb = sys.exc_info()[2]
1036         exc.recipe = filename
1037         exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
1038         raise exc
1039     # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
1040     # and for example a worker thread doesn't just exit on its own in response to
1041     # a SystemExit event for example.
1042     except BaseException, exc:
1043         raise ParsingFailure(exc, filename)
1044
1045 class CookerParser(object):
1046     def __init__(self, cooker, filelist, masked):
1047         self.filelist = filelist
1048         self.cooker = cooker
1049         self.cfgdata = cooker.configuration.data
1050
1051         # Accounting statistics
1052         self.parsed = 0
1053         self.cached = 0
1054         self.error = 0
1055         self.masked = masked
1056
1057         self.skipped = 0
1058         self.virtuals = 0
1059         self.total = len(filelist)
1060
1061         self.current = 0
1062         self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
1063                                  multiprocessing.cpu_count())
1064
1065         self.bb_cache = bb.cache.Cache(self.cfgdata)
1066         self.fromcache = []
1067         self.willparse = []
1068         for filename in self.filelist:
1069             appends = self.cooker.get_file_appends(filename)
1070             if not self.bb_cache.cacheValid(filename):
1071                 self.willparse.append((filename, appends))
1072             else:
1073                 self.fromcache.append((filename, appends))
1074         self.toparse = self.total - len(self.fromcache)
1075         self.progress_chunk = max(self.toparse / 100, 1)
1076
1077         self.start()
1078
1079     def start(self):
1080         def init(cfg):
1081             parse_file.cfg = cfg
1082             multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
1083
1084         self.results = self.load_cached()
1085
1086         if self.toparse:
1087             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
1088
1089             self.pool = multiprocessing.Pool(self.num_processes, init, [self.cfgdata])
1090             parsed = self.pool.imap(parse_file, self.willparse)
1091             self.pool.close()
1092
1093             self.results = itertools.chain(self.results, parsed)
1094
1095     def shutdown(self, clean=True):
1096         if not self.toparse:
1097             return
1098
1099         if clean:
1100             event = bb.event.ParseCompleted(self.cached, self.parsed,
1101                                             self.skipped, self.masked,
1102                                             self.virtuals, self.error,
1103                                             self.total)
1104             bb.event.fire(event, self.cfgdata)
1105         else:
1106             self.pool.terminate()
1107         self.pool.join()
1108
1109         bb.codeparser.parser_cache_save(self.cfgdata)
1110
1111         sync = threading.Thread(target=self.bb_cache.sync)
1112         sync.start()
1113         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
1114
1115     def load_cached(self):
1116         for filename, appends in self.fromcache:
1117             cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)
1118             yield not cached, infos
1119
1120     def parse_next(self):
1121         try:
1122             parsed, result = self.results.next()
1123         except StopIteration:
1124             self.shutdown()
1125             return False
1126         except ParsingFailure as exc:
1127             self.shutdown(clean=False)
1128             bb.fatal('Error parsing %s: %s' %
1129                      (exc.recipe, bb.exceptions.to_string(exc.realexception)))
1130         except Exception:
1131             import traceback
1132             etype, value, tb = sys.exc_info()
1133             formatted = bb.exceptions.format_extracted(value.traceback, limit=5)
1134             formatted.extend(traceback.format_exception_only(etype, value))
1135
1136             self.shutdown(clean=False)
1137             bb.fatal('Error parsing %s:\n%s' % (value.recipe, ''.join(formatted)))
1138
1139         self.current += 1
1140         self.virtuals += len(result)
1141         if parsed:
1142             self.parsed += 1
1143             if self.parsed % self.progress_chunk == 0:
1144                 bb.event.fire(bb.event.ParseProgress(self.parsed),
1145                               self.cfgdata)
1146         else:
1147             self.cached += 1
1148
1149         for virtualfn, info in result:
1150             if info.skipped:
1151                 self.skipped += 1
1152             self.bb_cache.add_info(virtualfn, info, self.cooker.status,
1153                                         parsed=parsed)
1154         return True
1155
1156     def reparse(self, filename):
1157         infos = self.bb_cache.parse(filename,
1158                                     self.cooker.get_file_appends(filename),
1159                                     self.cfgdata)
1160         for vfn, info in infos:
1161             self.cooker.status.add_from_recipeinfo(vfn, info)