event.py: Add ConfigParsed Event after configuration parsing is complete
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
77         """
78         Build one task of a package, optionally build following task depends
79         """
80         bb.event.fire(bb.event.PkgStarted(item, the_data))
81         try:
82             if not build_depends:
83                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
84             if not self.configuration.dry_run:
85                 bb.build.exec_task('do_%s' % task, the_data)
86             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
87             return True
88         except bb.build.FuncFailed:
89             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
90             bb.event.fire(bb.event.PkgFailed(item, the_data))
91             raise
92         except bb.build.EventException, e:
93             event = e.args[1]
94             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
95             bb.event.fire(bb.event.PkgFailed(item, the_data))
96             raise
97
98     def tryBuild( self, fn, build_depends):
99         """
100         Build a provider and its dependencies. 
101         build_depends is a list of previous build dependencies (not runtime)
102         If build_depends is empty, we're dealing with a runtime depends
103         """
104
105         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
106
107         item = self.status.pkg_fn[fn]
108
109         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
110             return True
111
112         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
113
114     def showVersions(self):
115         pkg_pn = self.status.pkg_pn
116         preferred_versions = {}
117         latest_versions = {}
118
119         # Sort by priority
120         for pn in pkg_pn.keys():
121             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
122             preferred_versions[pn] = (pref_ver, pref_file)
123             latest_versions[pn] = (last_ver, last_file)
124
125         pkg_list = pkg_pn.keys()
126         pkg_list.sort()
127
128         for p in pkg_list:
129             pref = preferred_versions[p]
130             latest = latest_versions[p]
131
132             if pref != latest:
133                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
134             else:
135                 prefstr = ""
136
137             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
138                                         prefstr)
139
140
141     def showEnvironment( self ):
142         """Show the outer or per-package environment"""
143         if self.configuration.buildfile:
144             self.cb = None
145             self.bb_cache = bb.cache.init(self)
146             try:
147                 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
148             except IOError, e:
149                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
150             except Exception, e:
151                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
152         # emit variables and shell functions
153         try:
154             data.update_data( self.configuration.data )
155             data.emit_env(sys.__stdout__, self.configuration.data, True)
156         except Exception, e:
157             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
158         # emit the metadata which isnt valid shell
159         data.expandKeys( self.configuration.data )      
160         for e in self.configuration.data.keys():
161             if data.getVarFlag( e, 'python', self.configuration.data ):
162                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
163
164     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
165         """
166         Generate a task dependency graph. 
167
168         pkgs_to_build A list of packages that needs to be built
169         ignore_deps   A list of names where processing of dependencies
170                       should be stopped. e.g. dependencies that get
171         """
172
173         for dep in ignore_deps:
174             self.status.ignored_dependencies.add(dep)
175
176         localdata = data.createCopy(self.configuration.data)
177         bb.data.update_data(localdata)
178         bb.data.expandKeys(localdata)
179         taskdata = bb.taskdata.TaskData(self.configuration.abort)
180
181         runlist = []
182         try:
183             for k in pkgs_to_build:
184                 taskdata.add_provider(localdata, self.status, k)
185                 runlist.append([k, "do_%s" % self.configuration.cmd])
186             taskdata.add_unresolved(localdata, self.status)
187         except bb.providers.NoProvider:
188             sys.exit(1)
189         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
190         rq.prepare_runqueue()
191
192         seen_fnids = []  
193         depends_file = file('depends.dot', 'w' )
194         tdepends_file = file('task-depends.dot', 'w' )
195         print >> depends_file, "digraph depends {"
196         print >> tdepends_file, "digraph depends {"
197         rq.prio_map.reverse()
198         for task1 in range(len(rq.runq_fnid)):
199             task = rq.prio_map[task1]
200             taskname = rq.runq_task[task]
201             fnid = rq.runq_fnid[task]
202             fn = taskdata.fn_index[fnid]
203             pn = self.status.pkg_fn[fn]
204             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
205             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
206             for dep in rq.runq_depends[task]:
207                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
208                 deppn = self.status.pkg_fn[depfn]
209                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
210             if fnid not in seen_fnids:
211                 seen_fnids.append(fnid)
212                 packages = []
213                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
214                 for depend in self.status.deps[fn]:
215                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
216                 rdepends = self.status.rundeps[fn]
217                 for package in rdepends:
218                     for rdepend in rdepends[package]:
219                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
220                     packages.append(package)
221                 rrecs = self.status.runrecs[fn]
222                 for package in rrecs:
223                     for rdepend in rrecs[package]:
224                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
225                     if not package in packages:
226                         packages.append(package)
227                 for package in packages:
228                     if package != pn:
229                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
230                         for depend in self.status.deps[fn]:
231                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
232                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
233                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
234                 #for rdep in taskdata.rdepids[fnid]:
235                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
236                 #for dep in taskdata.depids[fnid]:
237                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
238         print >> depends_file,  "}"
239         print >> tdepends_file,  "}"
240         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
241         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
242
243     def buildDepgraph( self ):
244         all_depends = self.status.all_depends
245         pn_provides = self.status.pn_provides
246
247         localdata = data.createCopy(self.configuration.data)
248         bb.data.update_data(localdata)
249         bb.data.expandKeys(localdata)
250
251         def calc_bbfile_priority(filename):
252             for (regex, pri) in self.status.bbfile_config_priorities:
253                 if regex.match(filename):
254                     return pri
255             return 0
256
257         # Handle PREFERRED_PROVIDERS
258         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
259             (providee, provider) = p.split(':')
260             if providee in self.status.preferred and self.status.preferred[providee] != provider:
261                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
262             self.status.preferred[providee] = provider
263
264         # Calculate priorities for each file
265         for p in self.status.pkg_fn.keys():
266             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
267
268     def buildWorldTargetList(self):
269         """
270          Build package list for "bitbake world"
271         """
272         all_depends = self.status.all_depends
273         pn_provides = self.status.pn_provides
274         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
275         for f in self.status.possible_world:
276             terminal = True
277             pn = self.status.pkg_fn[f]
278
279             for p in pn_provides[pn]:
280                 if p.startswith('virtual/'):
281                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
282                     terminal = False
283                     break
284                 for pf in self.status.providers[p]:
285                     if self.status.pkg_fn[pf] != pn:
286                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
287                         terminal = False
288                         break
289             if terminal:
290                 self.status.world_target.add(pn)
291
292             # drop reference count now
293             self.status.possible_world = None
294             self.status.all_depends    = None
295
296     def myProgressCallback( self, x, y, f, from_cache ):
297         """Update any tty with the progress change"""
298         if os.isatty(sys.stdout.fileno()):
299             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
300             sys.stdout.flush()
301         else:
302             if x == 1:
303                 sys.stdout.write("Parsing .bb files, please wait...")
304                 sys.stdout.flush()
305             if x == y:
306                 sys.stdout.write("done.")
307                 sys.stdout.flush()
308
309     def interactiveMode( self ):
310         """Drop off into a shell"""
311         try:
312             from bb import shell
313         except ImportError, details:
314             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
315         else:
316             bb.data.update_data( self.configuration.data )
317             bb.data.expandKeys( self.configuration.data )
318             shell.start( self )
319             sys.exit( 0 )
320
321     def parseConfigurationFile( self, afile ):
322         try:
323             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
324
325             # Add the handlers we inherited by INHERIT
326             # we need to do this manually as it is not guranteed
327             # we will pick up these classes... as we only INHERIT
328             # on .inc and .bb files but not on .conf
329             data = bb.data.createCopy( self.configuration.data )
330             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
331             for inherit in inherits:
332                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
333
334             # FIXME: This assumes that we included at least one .inc file
335             for var in bb.data.keys(data):
336                 if bb.data.getVarFlag(var, 'handler', data):
337                     bb.event.register(var,bb.data.getVar(var, data))
338
339             bb.fetch.fetcher_init(self.configuration.data)
340
341             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
342
343         except IOError:
344             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
345         except bb.parse.ParseError, details:
346             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
347
348     def handleCollections( self, collections ):
349         """Handle collections"""
350         if collections:
351             collection_list = collections.split()
352             for c in collection_list:
353                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
354                 if regex == None:
355                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
356                     continue
357                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
358                 if priority == None:
359                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
360                     continue
361                 try:
362                     cre = re.compile(regex)
363                 except re.error:
364                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
365                     continue
366                 try:
367                     pri = int(priority)
368                     self.status.bbfile_config_priorities.append((cre, pri))
369                 except ValueError:
370                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
371
372     def buildSetVars(self):
373         """
374         Setup any variables needed before starting a build
375         """
376         if not bb.data.getVar("BUILDNAME", self.configuration.data):
377             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
378         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
379
380     def buildFile(self, buildfile):
381         """
382         Build the file matching regexp buildfile
383         """
384
385         bf = os.path.abspath(buildfile)
386         try:
387             os.stat(bf)
388         except OSError:
389             (filelist, masked) = self.collect_bbfiles()
390             regexp = re.compile(buildfile)
391             matches = []
392             for f in filelist:
393                 if regexp.search(f) and os.path.isfile(f):
394                     bf = f
395                     matches.append(f)
396             if len(matches) != 1:
397                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
398                 for f in matches:
399                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
400                 sys.exit(1)
401             bf = matches[0]                 
402
403         bbfile_data = bb.parse.handle(bf, self.configuration.data)
404
405         # Remove stamp for target if force mode active
406         if self.configuration.force:
407             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
408             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
409
410         item = bb.data.getVar('PN', bbfile_data, 1)
411         try:
412             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
413         except bb.build.EventException:
414             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
415
416         sys.exit(0)
417
418     def buildTargets(self, targets):
419         """
420         Attempt to build the targets specified
421         """
422
423         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
424         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
425
426         localdata = data.createCopy(self.configuration.data)
427         bb.data.update_data(localdata)
428         bb.data.expandKeys(localdata)
429
430         taskdata = bb.taskdata.TaskData(self.configuration.abort)
431
432         runlist = []
433         try:
434             for k in targets:
435                 taskdata.add_provider(localdata, self.status, k)
436                 runlist.append([k, "do_%s" % self.configuration.cmd])
437             taskdata.add_unresolved(localdata, self.status)
438         except bb.providers.NoProvider:
439             sys.exit(1)
440
441         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
442         rq.prepare_runqueue()
443         try:
444             failures = rq.execute_runqueue()
445         except runqueue.TaskFailure, fnids:
446             for fnid in fnids:
447                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
448             sys.exit(1)
449         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
450
451         sys.exit(0)
452
453     def updateCache(self):
454         # Import Psyco if available and not disabled
455         if not self.configuration.disable_psyco:
456             try:
457                 import psyco
458             except ImportError:
459                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
460             else:
461                 psyco.bind( self.parse_bbfiles )
462         else:
463             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
464
465         self.status = bb.cache.CacheData()
466
467         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
468         self.status.ignored_dependencies = Set( ignore.split() )
469
470         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
471
472         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
473         (filelist, masked) = self.collect_bbfiles()
474         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
475         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
476
477         self.buildDepgraph()
478
479     def cook(self):
480         """
481         We are building stuff here. We do the building
482         from here. By default we try to execute task
483         build.
484         """
485
486         if self.configuration.show_environment:
487             self.showEnvironment()
488             sys.exit( 0 )
489
490         self.buildSetVars()
491
492         if self.configuration.interactive:
493             self.interactiveMode()
494
495         if self.configuration.buildfile is not None:
496             return self.buildFile(self.configuration.buildfile)
497
498         # initialise the parsing status now we know we will need deps
499         self.updateCache()
500
501         if self.configuration.parse_only:
502             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
503             return 0
504
505         pkgs_to_build = self.configuration.pkgs_to_build
506
507         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
508         if bbpkgs:
509             pkgs_to_build.extend(bbpkgs.split())
510         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
511                              and not self.configuration.show_environment:
512                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
513                 print "for usage information."
514                 sys.exit(0)
515
516         try:
517             if self.configuration.show_versions:
518                 self.showVersions()
519                 sys.exit( 0 )
520             if 'world' in pkgs_to_build:
521                 self.buildWorldTargetList()
522                 pkgs_to_build.remove('world')
523                 for t in self.status.world_target:
524                     pkgs_to_build.append(t)
525
526             if self.configuration.dot_graph:
527                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
528                 sys.exit( 0 )
529
530             return self.buildTargets(pkgs_to_build)
531
532         except KeyboardInterrupt:
533             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
534             sys.exit(1)
535
536     def get_bbfiles( self, path = os.getcwd() ):
537         """Get list of default .bb files by reading out the current directory"""
538         contents = os.listdir(path)
539         bbfiles = []
540         for f in contents:
541             (root, ext) = os.path.splitext(f)
542             if ext == ".bb":
543                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
544         return bbfiles
545
546     def find_bbfiles( self, path ):
547         """Find all the .bb files in a directory"""
548         from os.path import join
549
550         found = []
551         for dir, dirs, files in os.walk(path):
552             for ignored in ('SCCS', 'CVS', '.svn'):
553                 if ignored in dirs:
554                     dirs.remove(ignored)
555             found += [join(dir,f) for f in files if f.endswith('.bb')]
556
557         return found
558
559     def collect_bbfiles( self ):
560         """Collect all available .bb build files"""
561         parsed, cached, skipped, masked = 0, 0, 0, 0
562         self.bb_cache = bb.cache.init(self)
563
564         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
565         data.setVar("BBFILES", " ".join(files), self.configuration.data)
566
567         if not len(files):
568             files = self.get_bbfiles()
569
570         if not len(files):
571             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
572
573         newfiles = []
574         for f in files:
575             if os.path.isdir(f):
576                 dirfiles = self.find_bbfiles(f)
577                 if dirfiles:
578                     newfiles += dirfiles
579                     continue
580             newfiles += glob.glob(f) or [ f ]
581
582         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
583
584         if not bbmask:
585             return (newfiles, 0)
586
587         try:
588             bbmask_compiled = re.compile(bbmask)
589         except sre_constants.error:
590             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
591
592         finalfiles = []
593         for i in xrange( len( newfiles ) ):
594             f = newfiles[i]
595             if bbmask and bbmask_compiled.search(f):
596                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
597                 masked += 1
598                 continue
599             finalfiles.append(f)
600
601         return (finalfiles, masked)
602
603     def parse_bbfiles(self, filelist, masked, progressCallback = None):
604         parsed, cached, skipped, error = 0, 0, 0, 0
605         for i in xrange( len( filelist ) ):
606             f = filelist[i]
607
608             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
609
610             # read a file's metadata
611             try:
612                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
613                 if skip:
614                     skipped += 1
615                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
616                     self.bb_cache.skip(f)
617                     continue
618                 elif fromCache: cached += 1
619                 else: parsed += 1
620                 deps = None
621
622                 # Disabled by RP as was no longer functional
623                 # allow metadata files to add items to BBFILES
624                 #data.update_data(self.pkgdata[f])
625                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
626                 #if addbbfiles:
627                 #    for aof in addbbfiles.split():
628                 #        if not files.count(aof):
629                 #            if not os.path.isabs(aof):
630                 #                aof = os.path.join(os.path.dirname(f),aof)
631                 #            files.append(aof)
632
633                 self.bb_cache.handle_data(f, self.status)
634
635                 # now inform the caller
636                 if progressCallback is not None:
637                     progressCallback( i + 1, len( filelist ), f, fromCache )
638
639             except IOError, e:
640                 self.bb_cache.remove(f)
641                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
642                 pass
643             except KeyboardInterrupt:
644                 self.bb_cache.sync()
645                 raise
646             except Exception, e:
647                 error += 1
648                 self.bb_cache.remove(f)
649                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
650             except:
651                 self.bb_cache.remove(f)
652                 raise
653
654         if progressCallback is not None:
655             print "\r" # need newline after Handling Bitbake files message
656             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
657
658         self.bb_cache.sync()
659
660         if error > 0:
661             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")