cooker.py: Make sure Build Completed events are generated even when tasks fail
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
71         if bbpkgs:
72             self.configuration.pkgs_to_build.extend(bbpkgs.split())
73
74         #
75         # Special updated configuration we use for firing events
76         #
77         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
78         bb.data.update_data(self.configuration.event_data)
79
80         #
81         # TOSTOP must not be set or our children will hang when they output
82         #
83         fd = sys.stdout.fileno()
84         if os.isatty(fd):
85             import termios
86             tcattr = termios.tcgetattr(fd)
87             if tcattr[3] & termios.TOSTOP:
88                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
89                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
90                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
91
92         # Change nice level if we're asked to
93         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
94         if nice:
95             curnice = os.nice(0)
96             nice = int(nice) - curnice
97             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
98  
99
100     def tryBuildPackage(self, fn, item, task, the_data):
101         """
102         Build one task of a package, optionally build following task depends
103         """
104         bb.event.fire(bb.event.PkgStarted(item, the_data))
105         try:
106             if not self.configuration.dry_run:
107                 bb.build.exec_task('do_%s' % task, the_data)
108             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
109             return True
110         except bb.build.FuncFailed:
111             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
112             bb.event.fire(bb.event.PkgFailed(item, the_data))
113             raise
114         except bb.build.EventException, e:
115             event = e.args[1]
116             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
117             bb.event.fire(bb.event.PkgFailed(item, the_data))
118             raise
119
120     def tryBuild(self, fn):
121         """
122         Build a provider and its dependencies. 
123         build_depends is a list of previous build dependencies (not runtime)
124         If build_depends is empty, we're dealing with a runtime depends
125         """
126         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
127
128         item = self.status.pkg_fn[fn]
129
130         #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
131         #    return True
132
133         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data)
134
135     def showVersions(self):
136         pkg_pn = self.status.pkg_pn
137         preferred_versions = {}
138         latest_versions = {}
139
140         # Sort by priority
141         for pn in pkg_pn.keys():
142             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
143             preferred_versions[pn] = (pref_ver, pref_file)
144             latest_versions[pn] = (last_ver, last_file)
145
146         pkg_list = pkg_pn.keys()
147         pkg_list.sort()
148
149         for p in pkg_list:
150             pref = preferred_versions[p]
151             latest = latest_versions[p]
152
153             if pref != latest:
154                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
155             else:
156                 prefstr = ""
157
158             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
159                                         prefstr)
160
161
162     def showEnvironment(self , buildfile = None, pkgs_to_build = []):
163         """
164         Show the outer or per-package environment
165         """
166         fn = None
167         envdata = None
168
169         if 'world' in pkgs_to_build:
170             print "'world' is not a valid target for --environment."
171             sys.exit(1)
172
173         if len(pkgs_to_build) > 1:
174             print "Only one target can be used with the --environment option."
175             sys.exit(1)
176
177         if buildfile:
178             if len(pkgs_to_build) > 0:
179                 print "No target should be used with the --environment and --buildfile options."
180                 sys.exit(1)
181             self.cb = None
182             self.bb_cache = bb.cache.init(self)
183             fn = self.matchFile(buildfile)
184             if not fn:
185                 sys.exit(1)
186         elif len(pkgs_to_build) == 1:
187             self.updateCache()
188
189             localdata = data.createCopy(self.configuration.data)
190             bb.data.update_data(localdata)
191             bb.data.expandKeys(localdata)
192
193             taskdata = bb.taskdata.TaskData(self.configuration.abort)
194
195             try:
196                 taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
197                 taskdata.add_unresolved(localdata, self.status)
198             except bb.providers.NoProvider:
199                 sys.exit(1)
200
201             targetid = taskdata.getbuild_id(pkgs_to_build[0])
202             fnid = taskdata.build_targets[targetid][0]
203             fn = taskdata.fn_index[fnid]
204         else:
205             envdata = self.configuration.data
206
207         if fn:
208             try:
209                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
210             except IOError, e:
211                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
212             except Exception, e:
213                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
214
215         # emit variables and shell functions
216         try:
217             data.update_data( envdata )
218             data.emit_env(sys.__stdout__, envdata, True)
219         except Exception, e:
220             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
221         # emit the metadata which isnt valid shell
222         data.expandKeys( envdata )
223         for e in envdata.keys():
224             if data.getVarFlag( e, 'python', envdata ):
225                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
226
227     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
228         """
229         Generate a task dependency graph. 
230
231         pkgs_to_build A list of packages that needs to be built
232         ignore_deps   A list of names where processing of dependencies
233                       should be stopped. e.g. dependencies that get
234         """
235
236         for dep in ignore_deps:
237             self.status.ignored_dependencies.add(dep)
238
239         localdata = data.createCopy(self.configuration.data)
240         bb.data.update_data(localdata)
241         bb.data.expandKeys(localdata)
242         taskdata = bb.taskdata.TaskData(self.configuration.abort)
243
244         runlist = []
245         try:
246             for k in pkgs_to_build:
247                 taskdata.add_provider(localdata, self.status, k)
248                 runlist.append([k, "do_%s" % self.configuration.cmd])
249             taskdata.add_unresolved(localdata, self.status)
250         except bb.providers.NoProvider:
251             sys.exit(1)
252         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
253         rq.prepare_runqueue()
254
255         seen_fnids = []  
256         depends_file = file('depends.dot', 'w' )
257         tdepends_file = file('task-depends.dot', 'w' )
258         print >> depends_file, "digraph depends {"
259         print >> tdepends_file, "digraph depends {"
260
261         for task in range(len(rq.runq_fnid)):
262             taskname = rq.runq_task[task]
263             fnid = rq.runq_fnid[task]
264             fn = taskdata.fn_index[fnid]
265             pn = self.status.pkg_fn[fn]
266             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
267             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
268             for dep in rq.runq_depends[task]:
269                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
270                 deppn = self.status.pkg_fn[depfn]
271                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
272             if fnid not in seen_fnids:
273                 seen_fnids.append(fnid)
274                 packages = []
275                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
276                 for depend in self.status.deps[fn]:
277                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
278                 rdepends = self.status.rundeps[fn]
279                 for package in rdepends:
280                     for rdepend in rdepends[package]:
281                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
282                     packages.append(package)
283                 rrecs = self.status.runrecs[fn]
284                 for package in rrecs:
285                     for rdepend in rrecs[package]:
286                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
287                     if not package in packages:
288                         packages.append(package)
289                 for package in packages:
290                     if package != pn:
291                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
292                         for depend in self.status.deps[fn]:
293                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
294                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
295                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
296                 #for rdep in taskdata.rdepids[fnid]:
297                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
298                 #for dep in taskdata.depids[fnid]:
299                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
300         print >> depends_file,  "}"
301         print >> tdepends_file,  "}"
302         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
303         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
304
305     def buildDepgraph( self ):
306         all_depends = self.status.all_depends
307         pn_provides = self.status.pn_provides
308
309         localdata = data.createCopy(self.configuration.data)
310         bb.data.update_data(localdata)
311         bb.data.expandKeys(localdata)
312
313         def calc_bbfile_priority(filename):
314             for (regex, pri) in self.status.bbfile_config_priorities:
315                 if regex.match(filename):
316                     return pri
317             return 0
318
319         # Handle PREFERRED_PROVIDERS
320         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
321             try:
322                 (providee, provider) = p.split(':')
323             except:
324                 bb.msg.error(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
325                 continue
326             if providee in self.status.preferred and self.status.preferred[providee] != provider:
327                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
328             self.status.preferred[providee] = provider
329
330         # Calculate priorities for each file
331         for p in self.status.pkg_fn.keys():
332             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
333
334     def buildWorldTargetList(self):
335         """
336          Build package list for "bitbake world"
337         """
338         all_depends = self.status.all_depends
339         pn_provides = self.status.pn_provides
340         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
341         for f in self.status.possible_world:
342             terminal = True
343             pn = self.status.pkg_fn[f]
344
345             for p in pn_provides[pn]:
346                 if p.startswith('virtual/'):
347                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
348                     terminal = False
349                     break
350                 for pf in self.status.providers[p]:
351                     if self.status.pkg_fn[pf] != pn:
352                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
353                         terminal = False
354                         break
355             if terminal:
356                 self.status.world_target.add(pn)
357
358             # drop reference count now
359             self.status.possible_world = None
360             self.status.all_depends    = None
361
362     def myProgressCallback( self, x, y, f, from_cache ):
363         """Update any tty with the progress change"""
364         if os.isatty(sys.stdout.fileno()):
365             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
366             sys.stdout.flush()
367         else:
368             if x == 1:
369                 sys.stdout.write("Parsing .bb files, please wait...")
370                 sys.stdout.flush()
371             if x == y:
372                 sys.stdout.write("done.")
373                 sys.stdout.flush()
374
375     def interactiveMode( self ):
376         """Drop off into a shell"""
377         try:
378             from bb import shell
379         except ImportError, details:
380             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
381         else:
382             shell.start( self )
383             sys.exit( 0 )
384
385     def parseConfigurationFile( self, afile ):
386         try:
387             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
388
389             # Handle any INHERITs and inherit the base class
390             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
391             for inherit in inherits:
392                 self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
393
394             # Nomally we only register event handlers at the end of parsing .bb files
395             # We register any handlers we've found so far here...
396             for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
397                 bb.event.register(var,bb.data.getVar(var, self.configuration.data))
398
399             bb.fetch.fetcher_init(self.configuration.data)
400
401             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
402
403         except IOError:
404             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
405         except bb.parse.ParseError, details:
406             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
407
408     def handleCollections( self, collections ):
409         """Handle collections"""
410         if collections:
411             collection_list = collections.split()
412             for c in collection_list:
413                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
414                 if regex == None:
415                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
416                     continue
417                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
418                 if priority == None:
419                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
420                     continue
421                 try:
422                     cre = re.compile(regex)
423                 except re.error:
424                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
425                     continue
426                 try:
427                     pri = int(priority)
428                     self.status.bbfile_config_priorities.append((cre, pri))
429                 except ValueError:
430                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
431
432     def buildSetVars(self):
433         """
434         Setup any variables needed before starting a build
435         """
436         if not bb.data.getVar("BUILDNAME", self.configuration.data):
437             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
438         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
439
440     def matchFile(self, buildfile):
441         """
442         Convert the fragment buildfile into a real file
443         Error if there are too many matches
444         """
445         bf = os.path.abspath(buildfile)
446         try:
447             os.stat(bf)
448             return bf
449         except OSError:
450             (filelist, masked) = self.collect_bbfiles()
451             regexp = re.compile(buildfile)
452             matches = []
453             for f in filelist:
454                 if regexp.search(f) and os.path.isfile(f):
455                     bf = f
456                     matches.append(f)
457             if len(matches) != 1:
458                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
459                 for f in matches:
460                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
461                 return False
462             return matches[0]
463
464     def buildFile(self, buildfile):
465         """
466         Build the file matching regexp buildfile
467         """
468
469         # Make sure our target is a fully qualified filename
470         fn = self.matchFile(buildfile)
471         if not fn:
472             return False
473
474         # Load data into the cache for fn
475         self.bb_cache = bb.cache.init(self)
476         self.bb_cache.loadData(fn, self.configuration.data)
477
478         # Parse the loaded cache data
479         self.status = bb.cache.CacheData()
480         self.bb_cache.handle_data(fn, self.status)
481
482         # Tweak some variables
483         item = self.bb_cache.getVar('PN', fn, True)
484         self.status.ignored_dependencies = Set()
485         self.status.bbfile_priority[fn] = 1
486
487         # Remove external dependencies
488         self.status.task_deps[fn]['depends'] = {}
489         self.status.deps[fn] = []
490         self.status.rundeps[fn] = []
491         self.status.runrecs[fn] = []
492
493         # Remove stamp for target if force mode active
494         if self.configuration.force:
495             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, fn))
496             bb.build.del_stamp('do_%s' % self.configuration.cmd, self.configuration.data)
497
498         # Setup taskdata structure
499         taskdata = bb.taskdata.TaskData(self.configuration.abort)
500         taskdata.add_provider(self.configuration.data, self.status, item)
501
502         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
503         bb.event.fire(bb.event.BuildStarted(buildname, [item], self.configuration.event_data))
504
505         # Execute the runqueue
506         runlist = [[item, "do_%s" % self.configuration.cmd]]
507         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
508         rq.prepare_runqueue()
509         try:
510             failures = rq.execute_runqueue()
511         except runqueue.TaskFailure, fnids:
512             failures = 0
513             for fnid in fnids:
514                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
515                 failures = failures + 1
516             bb.event.fire(bb.event.BuildCompleted(buildname, [item], self.configuration.event_data, failures))
517             return False
518         bb.event.fire(bb.event.BuildCompleted(buildname, [item], self.configuration.event_data, failures))
519         return True
520
521     def buildTargets(self, targets):
522         """
523         Attempt to build the targets specified
524         """
525
526         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
527         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
528
529         localdata = data.createCopy(self.configuration.data)
530         bb.data.update_data(localdata)
531         bb.data.expandKeys(localdata)
532
533         taskdata = bb.taskdata.TaskData(self.configuration.abort)
534
535         runlist = []
536         try:
537             for k in targets:
538                 taskdata.add_provider(localdata, self.status, k)
539                 runlist.append([k, "do_%s" % self.configuration.cmd])
540             taskdata.add_unresolved(localdata, self.status)
541         except bb.providers.NoProvider:
542             sys.exit(1)
543
544         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
545         rq.prepare_runqueue()
546         try:
547             failures = rq.execute_runqueue()
548         except runqueue.TaskFailure, fnids:
549             failures = 0
550             for fnid in fnids:
551                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
552                 failures = failures + 1
553             bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
554             sys.exit(1)
555         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
556
557         sys.exit(0)
558
559     def updateCache(self):
560         # Import Psyco if available and not disabled
561         import platform
562         if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
563             if not self.configuration.disable_psyco:
564                 try:
565                     import psyco
566                 except ImportError:
567                     bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
568                 else:
569                     psyco.bind( self.parse_bbfiles )
570             else:
571                 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
572
573         self.status = bb.cache.CacheData()
574
575         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
576         self.status.ignored_dependencies = Set( ignore.split() )
577
578         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
579
580         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
581         (filelist, masked) = self.collect_bbfiles()
582         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
583         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
584
585         self.buildDepgraph()
586
587     def cook(self):
588         """
589         We are building stuff here. We do the building
590         from here. By default we try to execute task
591         build.
592         """
593
594         if self.configuration.show_environment:
595             self.showEnvironment(self.configuration.buildfile, self.configuration.pkgs_to_build)
596             sys.exit( 0 )
597
598         self.buildSetVars()
599
600         if self.configuration.interactive:
601             self.interactiveMode()
602
603         if self.configuration.buildfile is not None:
604             if not self.buildFile(self.configuration.buildfile):
605                 sys.exit(1)
606             sys.exit(0)
607
608         # initialise the parsing status now we know we will need deps
609         self.updateCache()
610
611         if self.configuration.parse_only:
612             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
613             return 0
614
615         pkgs_to_build = self.configuration.pkgs_to_build
616
617         if len(pkgs_to_build) == 0 and not self.configuration.show_versions:
618                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
619                 print "for usage information."
620                 sys.exit(0)
621
622         try:
623             if self.configuration.show_versions:
624                 self.showVersions()
625                 sys.exit( 0 )
626             if 'world' in pkgs_to_build:
627                 self.buildWorldTargetList()
628                 pkgs_to_build.remove('world')
629                 for t in self.status.world_target:
630                     pkgs_to_build.append(t)
631
632             if self.configuration.dot_graph:
633                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
634                 sys.exit( 0 )
635
636             return self.buildTargets(pkgs_to_build)
637
638         except KeyboardInterrupt:
639             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
640             sys.exit(1)
641
642     def get_bbfiles( self, path = os.getcwd() ):
643         """Get list of default .bb files by reading out the current directory"""
644         contents = os.listdir(path)
645         bbfiles = []
646         for f in contents:
647             (root, ext) = os.path.splitext(f)
648             if ext == ".bb":
649                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
650         return bbfiles
651
652     def find_bbfiles( self, path ):
653         """Find all the .bb files in a directory"""
654         from os.path import join
655
656         found = []
657         for dir, dirs, files in os.walk(path):
658             for ignored in ('SCCS', 'CVS', '.svn'):
659                 if ignored in dirs:
660                     dirs.remove(ignored)
661             found += [join(dir,f) for f in files if f.endswith('.bb')]
662
663         return found
664
665     def collect_bbfiles( self ):
666         """Collect all available .bb build files"""
667         parsed, cached, skipped, masked = 0, 0, 0, 0
668         self.bb_cache = bb.cache.init(self)
669
670         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
671         data.setVar("BBFILES", " ".join(files), self.configuration.data)
672
673         if not len(files):
674             files = self.get_bbfiles()
675
676         if not len(files):
677             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
678
679         newfiles = []
680         for f in files:
681             if os.path.isdir(f):
682                 dirfiles = self.find_bbfiles(f)
683                 if dirfiles:
684                     newfiles += dirfiles
685                     continue
686             newfiles += glob.glob(f) or [ f ]
687
688         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
689
690         if not bbmask:
691             return (newfiles, 0)
692
693         try:
694             bbmask_compiled = re.compile(bbmask)
695         except sre_constants.error:
696             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
697
698         finalfiles = []
699         for i in xrange( len( newfiles ) ):
700             f = newfiles[i]
701             if bbmask and bbmask_compiled.search(f):
702                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
703                 masked += 1
704                 continue
705             finalfiles.append(f)
706
707         return (finalfiles, masked)
708
709     def parse_bbfiles(self, filelist, masked, progressCallback = None):
710         parsed, cached, skipped, error = 0, 0, 0, 0
711         for i in xrange( len( filelist ) ):
712             f = filelist[i]
713
714             #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
715
716             # read a file's metadata
717             try:
718                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
719                 if skip:
720                     skipped += 1
721                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
722                     self.bb_cache.skip(f)
723                     continue
724                 elif fromCache: cached += 1
725                 else: parsed += 1
726                 deps = None
727
728                 # Disabled by RP as was no longer functional
729                 # allow metadata files to add items to BBFILES
730                 #data.update_data(self.pkgdata[f])
731                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
732                 #if addbbfiles:
733                 #    for aof in addbbfiles.split():
734                 #        if not files.count(aof):
735                 #            if not os.path.isabs(aof):
736                 #                aof = os.path.join(os.path.dirname(f),aof)
737                 #            files.append(aof)
738
739                 self.bb_cache.handle_data(f, self.status)
740
741                 # now inform the caller
742                 if progressCallback is not None:
743                     progressCallback( i + 1, len( filelist ), f, fromCache )
744
745             except IOError, e:
746                 self.bb_cache.remove(f)
747                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
748                 pass
749             except KeyboardInterrupt:
750                 self.bb_cache.sync()
751                 raise
752             except Exception, e:
753                 error += 1
754                 self.bb_cache.remove(f)
755                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
756             except:
757                 self.bb_cache.remove(f)
758                 raise
759
760         if progressCallback is not None:
761             print "\r" # need newline after Handling Bitbake files message
762             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
763
764         self.bb_cache.sync()
765
766         if error > 0:
767             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")