shell.py: Update to use cooker.buildFile()
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
71         if bbpkgs:
72             self.configuration.pkgs_to_build.extend(bbpkgs.split())
73
74         #
75         # Special updated configuration we use for firing events
76         #
77         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
78         bb.data.update_data(self.configuration.event_data)
79
80         #
81         # TOSTOP must not be set or our children will hang when they output
82         #
83         fd = sys.stdout.fileno()
84         if os.isatty(fd):
85             import termios
86             tcattr = termios.tcgetattr(fd)
87             if tcattr[3] & termios.TOSTOP:
88                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
89                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
90                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
91
92         # Change nice level if we're asked to
93         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
94         if nice:
95             curnice = os.nice(0)
96             nice = int(nice) - curnice
97             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
98  
99
100     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
101         """
102         Build one task of a package, optionally build following task depends
103         """
104         bb.event.fire(bb.event.PkgStarted(item, the_data))
105         try:
106             if not build_depends:
107                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
108             if not self.configuration.dry_run:
109                 bb.build.exec_task('do_%s' % task, the_data)
110             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
111             return True
112         except bb.build.FuncFailed:
113             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
114             bb.event.fire(bb.event.PkgFailed(item, the_data))
115             raise
116         except bb.build.EventException, e:
117             event = e.args[1]
118             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
119             bb.event.fire(bb.event.PkgFailed(item, the_data))
120             raise
121
122     def tryBuild( self, fn, build_depends):
123         """
124         Build a provider and its dependencies. 
125         build_depends is a list of previous build dependencies (not runtime)
126         If build_depends is empty, we're dealing with a runtime depends
127         """
128         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
129
130         item = self.status.pkg_fn[fn]
131
132         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
133             return True
134
135         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
136
137     def showVersions(self):
138         pkg_pn = self.status.pkg_pn
139         preferred_versions = {}
140         latest_versions = {}
141
142         # Sort by priority
143         for pn in pkg_pn.keys():
144             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
145             preferred_versions[pn] = (pref_ver, pref_file)
146             latest_versions[pn] = (last_ver, last_file)
147
148         pkg_list = pkg_pn.keys()
149         pkg_list.sort()
150
151         for p in pkg_list:
152             pref = preferred_versions[p]
153             latest = latest_versions[p]
154
155             if pref != latest:
156                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
157             else:
158                 prefstr = ""
159
160             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
161                                         prefstr)
162
163
164     def showEnvironment(self , buildfile = None, pkgs_to_build = []):
165         """
166         Show the outer or per-package environment
167         """
168         fn = None
169         envdata = None
170
171         if 'world' in pkgs_to_build:
172             print "'world' is not a valid target for --environment."
173             sys.exit(1)
174
175         if len(pkgs_to_build) > 1:
176             print "Only one target can be used with the --environment option."
177             sys.exit(1)
178
179         if buildfile:
180             if len(pkgs_to_build) > 0:
181                 print "No target should be used with the --environment and --buildfile options."
182                 sys.exit(1)
183             self.cb = None
184             self.bb_cache = bb.cache.init(self)
185             fn = self.matchFile(buildfile)
186             if not fn:
187                 sys.exit(1)
188         elif len(pkgs_to_build) == 1:
189             self.updateCache()
190
191             localdata = data.createCopy(self.configuration.data)
192             bb.data.update_data(localdata)
193             bb.data.expandKeys(localdata)
194
195             taskdata = bb.taskdata.TaskData(self.configuration.abort)
196
197             try:
198                 taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
199                 taskdata.add_unresolved(localdata, self.status)
200             except bb.providers.NoProvider:
201                 sys.exit(1)
202
203             targetid = taskdata.getbuild_id(pkgs_to_build[0])
204             fnid = taskdata.build_targets[targetid][0]
205             fn = taskdata.fn_index[fnid]
206         else:
207             envdata = self.configuration.data
208
209         if fn:
210             try:
211                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
212             except IOError, e:
213                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
214             except Exception, e:
215                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
216
217         # emit variables and shell functions
218         try:
219             data.update_data( envdata )
220             data.emit_env(sys.__stdout__, envdata, True)
221         except Exception, e:
222             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
223         # emit the metadata which isnt valid shell
224         data.expandKeys( envdata )
225         for e in envdata.keys():
226             if data.getVarFlag( e, 'python', envdata ):
227                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
228
229     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
230         """
231         Generate a task dependency graph. 
232
233         pkgs_to_build A list of packages that needs to be built
234         ignore_deps   A list of names where processing of dependencies
235                       should be stopped. e.g. dependencies that get
236         """
237
238         for dep in ignore_deps:
239             self.status.ignored_dependencies.add(dep)
240
241         localdata = data.createCopy(self.configuration.data)
242         bb.data.update_data(localdata)
243         bb.data.expandKeys(localdata)
244         taskdata = bb.taskdata.TaskData(self.configuration.abort)
245
246         runlist = []
247         try:
248             for k in pkgs_to_build:
249                 taskdata.add_provider(localdata, self.status, k)
250                 runlist.append([k, "do_%s" % self.configuration.cmd])
251             taskdata.add_unresolved(localdata, self.status)
252         except bb.providers.NoProvider:
253             sys.exit(1)
254         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
255         rq.prepare_runqueue()
256
257         seen_fnids = []  
258         depends_file = file('depends.dot', 'w' )
259         tdepends_file = file('task-depends.dot', 'w' )
260         print >> depends_file, "digraph depends {"
261         print >> tdepends_file, "digraph depends {"
262
263         for task in range(len(rq.runq_fnid)):
264             taskname = rq.runq_task[task]
265             fnid = rq.runq_fnid[task]
266             fn = taskdata.fn_index[fnid]
267             pn = self.status.pkg_fn[fn]
268             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
269             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
270             for dep in rq.runq_depends[task]:
271                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
272                 deppn = self.status.pkg_fn[depfn]
273                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
274             if fnid not in seen_fnids:
275                 seen_fnids.append(fnid)
276                 packages = []
277                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
278                 for depend in self.status.deps[fn]:
279                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
280                 rdepends = self.status.rundeps[fn]
281                 for package in rdepends:
282                     for rdepend in rdepends[package]:
283                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
284                     packages.append(package)
285                 rrecs = self.status.runrecs[fn]
286                 for package in rrecs:
287                     for rdepend in rrecs[package]:
288                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
289                     if not package in packages:
290                         packages.append(package)
291                 for package in packages:
292                     if package != pn:
293                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
294                         for depend in self.status.deps[fn]:
295                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
296                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
297                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
298                 #for rdep in taskdata.rdepids[fnid]:
299                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
300                 #for dep in taskdata.depids[fnid]:
301                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
302         print >> depends_file,  "}"
303         print >> tdepends_file,  "}"
304         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
305         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
306
307     def buildDepgraph( self ):
308         all_depends = self.status.all_depends
309         pn_provides = self.status.pn_provides
310
311         localdata = data.createCopy(self.configuration.data)
312         bb.data.update_data(localdata)
313         bb.data.expandKeys(localdata)
314
315         def calc_bbfile_priority(filename):
316             for (regex, pri) in self.status.bbfile_config_priorities:
317                 if regex.match(filename):
318                     return pri
319             return 0
320
321         # Handle PREFERRED_PROVIDERS
322         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
323             try:
324                 (providee, provider) = p.split(':')
325             except:
326                 bb.msg.error(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
327                 continue
328             if providee in self.status.preferred and self.status.preferred[providee] != provider:
329                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
330             self.status.preferred[providee] = provider
331
332         # Calculate priorities for each file
333         for p in self.status.pkg_fn.keys():
334             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
335
336     def buildWorldTargetList(self):
337         """
338          Build package list for "bitbake world"
339         """
340         all_depends = self.status.all_depends
341         pn_provides = self.status.pn_provides
342         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
343         for f in self.status.possible_world:
344             terminal = True
345             pn = self.status.pkg_fn[f]
346
347             for p in pn_provides[pn]:
348                 if p.startswith('virtual/'):
349                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
350                     terminal = False
351                     break
352                 for pf in self.status.providers[p]:
353                     if self.status.pkg_fn[pf] != pn:
354                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
355                         terminal = False
356                         break
357             if terminal:
358                 self.status.world_target.add(pn)
359
360             # drop reference count now
361             self.status.possible_world = None
362             self.status.all_depends    = None
363
364     def myProgressCallback( self, x, y, f, from_cache ):
365         """Update any tty with the progress change"""
366         if os.isatty(sys.stdout.fileno()):
367             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
368             sys.stdout.flush()
369         else:
370             if x == 1:
371                 sys.stdout.write("Parsing .bb files, please wait...")
372                 sys.stdout.flush()
373             if x == y:
374                 sys.stdout.write("done.")
375                 sys.stdout.flush()
376
377     def interactiveMode( self ):
378         """Drop off into a shell"""
379         try:
380             from bb import shell
381         except ImportError, details:
382             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
383         else:
384             shell.start( self )
385             sys.exit( 0 )
386
387     def parseConfigurationFile( self, afile ):
388         try:
389             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
390
391             # Handle any INHERITs and inherit the base class
392             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
393             for inherit in inherits:
394                 self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
395
396             # Nomally we only register event handlers at the end of parsing .bb files
397             # We register any handlers we've found so far here...
398             for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
399                 bb.event.register(var,bb.data.getVar(var, self.configuration.data))
400
401             bb.fetch.fetcher_init(self.configuration.data)
402
403             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
404
405         except IOError:
406             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
407         except bb.parse.ParseError, details:
408             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
409
410     def handleCollections( self, collections ):
411         """Handle collections"""
412         if collections:
413             collection_list = collections.split()
414             for c in collection_list:
415                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
416                 if regex == None:
417                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
418                     continue
419                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
420                 if priority == None:
421                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
422                     continue
423                 try:
424                     cre = re.compile(regex)
425                 except re.error:
426                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
427                     continue
428                 try:
429                     pri = int(priority)
430                     self.status.bbfile_config_priorities.append((cre, pri))
431                 except ValueError:
432                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
433
434     def buildSetVars(self):
435         """
436         Setup any variables needed before starting a build
437         """
438         if not bb.data.getVar("BUILDNAME", self.configuration.data):
439             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
440         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
441
442     def matchFile(self, buildfile):
443         """
444         Convert the fragment buildfile into a real file
445         Error if there are too many matches
446         """
447         bf = os.path.abspath(buildfile)
448         try:
449             os.stat(bf)
450             return bf
451         except OSError:
452             (filelist, masked) = self.collect_bbfiles()
453             regexp = re.compile(buildfile)
454             matches = []
455             for f in filelist:
456                 if regexp.search(f) and os.path.isfile(f):
457                     bf = f
458                     matches.append(f)
459             if len(matches) != 1:
460                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
461                 for f in matches:
462                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
463                 return False
464             return matches[0]
465
466     def buildFile(self, buildfile):
467         """
468         Build the file matching regexp buildfile
469         """
470
471         # Make sure our target is a fully qualified filename
472         fn = self.matchFile(buildfile)
473         if not fn:
474             return False
475
476         bbfile_data = bb.parse.handle(fn, self.configuration.data)
477
478         # Remove stamp for target if force mode active
479         if self.configuration.force:
480             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, fn))
481             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
482
483         item = bb.data.getVar('PN', bbfile_data, 1)
484         try:
485             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
486         except bb.build.EventException:
487             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
488
489         return True
490
491     def buildTargets(self, targets):
492         """
493         Attempt to build the targets specified
494         """
495
496         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
497         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
498
499         localdata = data.createCopy(self.configuration.data)
500         bb.data.update_data(localdata)
501         bb.data.expandKeys(localdata)
502
503         taskdata = bb.taskdata.TaskData(self.configuration.abort)
504
505         runlist = []
506         try:
507             for k in targets:
508                 taskdata.add_provider(localdata, self.status, k)
509                 runlist.append([k, "do_%s" % self.configuration.cmd])
510             taskdata.add_unresolved(localdata, self.status)
511         except bb.providers.NoProvider:
512             sys.exit(1)
513
514         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
515         rq.prepare_runqueue()
516         try:
517             failures = rq.execute_runqueue()
518         except runqueue.TaskFailure, fnids:
519             for fnid in fnids:
520                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
521             sys.exit(1)
522         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
523
524         sys.exit(0)
525
526     def updateCache(self):
527         # Import Psyco if available and not disabled
528         import platform
529         if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
530             if not self.configuration.disable_psyco:
531                 try:
532                     import psyco
533                 except ImportError:
534                     bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
535                 else:
536                     psyco.bind( self.parse_bbfiles )
537             else:
538                 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
539
540         self.status = bb.cache.CacheData()
541
542         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
543         self.status.ignored_dependencies = Set( ignore.split() )
544
545         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
546
547         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
548         (filelist, masked) = self.collect_bbfiles()
549         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
550         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
551
552         self.buildDepgraph()
553
554     def cook(self):
555         """
556         We are building stuff here. We do the building
557         from here. By default we try to execute task
558         build.
559         """
560
561         if self.configuration.show_environment:
562             self.showEnvironment(self.configuration.buildfile, self.configuration.pkgs_to_build)
563             sys.exit( 0 )
564
565         self.buildSetVars()
566
567         if self.configuration.interactive:
568             self.interactiveMode()
569
570         if self.configuration.buildfile is not None:
571             if not self.buildFile(self.configuration.buildfile):
572                 sys.exit(1)
573             sys.exit(0)
574
575         # initialise the parsing status now we know we will need deps
576         self.updateCache()
577
578         if self.configuration.parse_only:
579             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
580             return 0
581
582         pkgs_to_build = self.configuration.pkgs_to_build
583
584         if len(pkgs_to_build) == 0 and not self.configuration.show_versions:
585                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
586                 print "for usage information."
587                 sys.exit(0)
588
589         try:
590             if self.configuration.show_versions:
591                 self.showVersions()
592                 sys.exit( 0 )
593             if 'world' in pkgs_to_build:
594                 self.buildWorldTargetList()
595                 pkgs_to_build.remove('world')
596                 for t in self.status.world_target:
597                     pkgs_to_build.append(t)
598
599             if self.configuration.dot_graph:
600                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
601                 sys.exit( 0 )
602
603             return self.buildTargets(pkgs_to_build)
604
605         except KeyboardInterrupt:
606             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
607             sys.exit(1)
608
609     def get_bbfiles( self, path = os.getcwd() ):
610         """Get list of default .bb files by reading out the current directory"""
611         contents = os.listdir(path)
612         bbfiles = []
613         for f in contents:
614             (root, ext) = os.path.splitext(f)
615             if ext == ".bb":
616                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
617         return bbfiles
618
619     def find_bbfiles( self, path ):
620         """Find all the .bb files in a directory"""
621         from os.path import join
622
623         found = []
624         for dir, dirs, files in os.walk(path):
625             for ignored in ('SCCS', 'CVS', '.svn'):
626                 if ignored in dirs:
627                     dirs.remove(ignored)
628             found += [join(dir,f) for f in files if f.endswith('.bb')]
629
630         return found
631
632     def collect_bbfiles( self ):
633         """Collect all available .bb build files"""
634         parsed, cached, skipped, masked = 0, 0, 0, 0
635         self.bb_cache = bb.cache.init(self)
636
637         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
638         data.setVar("BBFILES", " ".join(files), self.configuration.data)
639
640         if not len(files):
641             files = self.get_bbfiles()
642
643         if not len(files):
644             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
645
646         newfiles = []
647         for f in files:
648             if os.path.isdir(f):
649                 dirfiles = self.find_bbfiles(f)
650                 if dirfiles:
651                     newfiles += dirfiles
652                     continue
653             newfiles += glob.glob(f) or [ f ]
654
655         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
656
657         if not bbmask:
658             return (newfiles, 0)
659
660         try:
661             bbmask_compiled = re.compile(bbmask)
662         except sre_constants.error:
663             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
664
665         finalfiles = []
666         for i in xrange( len( newfiles ) ):
667             f = newfiles[i]
668             if bbmask and bbmask_compiled.search(f):
669                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
670                 masked += 1
671                 continue
672             finalfiles.append(f)
673
674         return (finalfiles, masked)
675
676     def parse_bbfiles(self, filelist, masked, progressCallback = None):
677         parsed, cached, skipped, error = 0, 0, 0, 0
678         for i in xrange( len( filelist ) ):
679             f = filelist[i]
680
681             #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
682
683             # read a file's metadata
684             try:
685                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
686                 if skip:
687                     skipped += 1
688                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
689                     self.bb_cache.skip(f)
690                     continue
691                 elif fromCache: cached += 1
692                 else: parsed += 1
693                 deps = None
694
695                 # Disabled by RP as was no longer functional
696                 # allow metadata files to add items to BBFILES
697                 #data.update_data(self.pkgdata[f])
698                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
699                 #if addbbfiles:
700                 #    for aof in addbbfiles.split():
701                 #        if not files.count(aof):
702                 #            if not os.path.isabs(aof):
703                 #                aof = os.path.join(os.path.dirname(f),aof)
704                 #            files.append(aof)
705
706                 self.bb_cache.handle_data(f, self.status)
707
708                 # now inform the caller
709                 if progressCallback is not None:
710                     progressCallback( i + 1, len( filelist ), f, fromCache )
711
712             except IOError, e:
713                 self.bb_cache.remove(f)
714                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
715                 pass
716             except KeyboardInterrupt:
717                 self.bb_cache.sync()
718                 raise
719             except Exception, e:
720                 error += 1
721                 self.bb_cache.remove(f)
722                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
723             except:
724                 self.bb_cache.remove(f)
725                 raise
726
727         if progressCallback is not None:
728             print "\r" # need newline after Handling Bitbake files message
729             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
730
731         self.bb_cache.sync()
732
733         if error > 0:
734             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")