Add persitent data store from trunk, sync the fetcher changes to use the persistent...
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
77         """
78         Build one task of a package, optionally build following task depends
79         """
80         bb.event.fire(bb.event.PkgStarted(item, the_data))
81         try:
82             if not build_depends:
83                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
84             if not self.configuration.dry_run:
85                 bb.build.exec_task('do_%s' % task, the_data)
86             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
87             return True
88         except bb.build.FuncFailed:
89             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
90             bb.event.fire(bb.event.PkgFailed(item, the_data))
91             raise
92         except bb.build.EventException, e:
93             event = e.args[1]
94             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
95             bb.event.fire(bb.event.PkgFailed(item, the_data))
96             raise
97
98     def tryBuild( self, fn, build_depends):
99         """
100         Build a provider and its dependencies. 
101         build_depends is a list of previous build dependencies (not runtime)
102         If build_depends is empty, we're dealing with a runtime depends
103         """
104
105         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
106
107         item = self.status.pkg_fn[fn]
108
109         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
110             return True
111
112         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
113
114     def showVersions(self):
115         pkg_pn = self.status.pkg_pn
116         preferred_versions = {}
117         latest_versions = {}
118
119         # Sort by priority
120         for pn in pkg_pn.keys():
121             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
122             preferred_versions[pn] = (pref_ver, pref_file)
123             latest_versions[pn] = (last_ver, last_file)
124
125         pkg_list = pkg_pn.keys()
126         pkg_list.sort()
127
128         for p in pkg_list:
129             pref = preferred_versions[p]
130             latest = latest_versions[p]
131
132             if pref != latest:
133                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
134             else:
135                 prefstr = ""
136
137             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
138                                         prefstr)
139
140
141     def showEnvironment( self ):
142         """Show the outer or per-package environment"""
143         if self.configuration.buildfile:
144             self.cb = None
145             self.bb_cache = bb.cache.init(self)
146             try:
147                 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
148             except IOError, e:
149                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
150             except Exception, e:
151                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
152         # emit variables and shell functions
153         try:
154             data.update_data( self.configuration.data )
155             data.emit_env(sys.__stdout__, self.configuration.data, True)
156         except Exception, e:
157             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
158         # emit the metadata which isnt valid shell
159         data.expandKeys( self.configuration.data )      
160         for e in self.configuration.data.keys():
161             if data.getVarFlag( e, 'python', self.configuration.data ):
162                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
163
164     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
165         """
166         Generate a task dependency graph. 
167
168         pkgs_to_build A list of packages that needs to be built
169         ignore_deps   A list of names where processing of dependencies
170                       should be stopped. e.g. dependencies that get
171         """
172
173         for dep in ignore_deps:
174             self.status.ignored_dependencies.add(dep)
175
176         localdata = data.createCopy(self.configuration.data)
177         bb.data.update_data(localdata)
178         bb.data.expandKeys(localdata)
179         taskdata = bb.taskdata.TaskData(self.configuration.abort)
180
181         runlist = []
182         try:
183             for k in pkgs_to_build:
184                 taskdata.add_provider(localdata, self.status, k)
185                 runlist.append([k, "do_%s" % self.configuration.cmd])
186             taskdata.add_unresolved(localdata, self.status)
187         except bb.providers.NoProvider:
188             sys.exit(1)
189         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
190         rq.prepare_runqueue()
191
192         seen_fnids = []  
193         depends_file = file('depends.dot', 'w' )
194         tdepends_file = file('task-depends.dot', 'w' )
195         print >> depends_file, "digraph depends {"
196         print >> tdepends_file, "digraph depends {"
197         rq.prio_map.reverse()
198         for task1 in range(len(rq.runq_fnid)):
199             task = rq.prio_map[task1]
200             taskname = rq.runq_task[task]
201             fnid = rq.runq_fnid[task]
202             fn = taskdata.fn_index[fnid]
203             pn = self.status.pkg_fn[fn]
204             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
205             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
206             for dep in rq.runq_depends[task]:
207                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
208                 deppn = self.status.pkg_fn[depfn]
209                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
210             if fnid not in seen_fnids:
211                 seen_fnids.append(fnid)
212                 packages = []
213                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
214                 for depend in self.status.deps[fn]:
215                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
216                 rdepends = self.status.rundeps[fn]
217                 for package in rdepends:
218                     for rdepend in rdepends[package]:
219                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
220                     packages.append(package)
221                 rrecs = self.status.runrecs[fn]
222                 for package in rrecs:
223                     for rdepend in rrecs[package]:
224                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
225                     if not package in packages:
226                         packages.append(package)
227                 for package in packages:
228                     if package != pn:
229                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
230                         for depend in self.status.deps[fn]:
231                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
232                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
233                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
234                 #for rdep in taskdata.rdepids[fnid]:
235                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
236                 #for dep in taskdata.depids[fnid]:
237                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
238         print >> depends_file,  "}"
239         print >> tdepends_file,  "}"
240         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
241         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
242
243     def buildDepgraph( self ):
244         all_depends = self.status.all_depends
245         pn_provides = self.status.pn_provides
246
247         localdata = data.createCopy(self.configuration.data)
248         bb.data.update_data(localdata)
249         bb.data.expandKeys(localdata)
250
251         def calc_bbfile_priority(filename):
252             for (regex, pri) in self.status.bbfile_config_priorities:
253                 if regex.match(filename):
254                     return pri
255             return 0
256
257         # Handle PREFERRED_PROVIDERS
258         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
259             (providee, provider) = p.split(':')
260             if providee in self.status.preferred and self.status.preferred[providee] != provider:
261                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
262             self.status.preferred[providee] = provider
263
264         # Calculate priorities for each file
265         for p in self.status.pkg_fn.keys():
266             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
267
268     def buildWorldTargetList(self):
269         """
270          Build package list for "bitbake world"
271         """
272         all_depends = self.status.all_depends
273         pn_provides = self.status.pn_provides
274         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
275         for f in self.status.possible_world:
276             terminal = True
277             pn = self.status.pkg_fn[f]
278
279             for p in pn_provides[pn]:
280                 if p.startswith('virtual/'):
281                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
282                     terminal = False
283                     break
284                 for pf in self.status.providers[p]:
285                     if self.status.pkg_fn[pf] != pn:
286                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
287                         terminal = False
288                         break
289             if terminal:
290                 self.status.world_target.add(pn)
291
292             # drop reference count now
293             self.status.possible_world = None
294             self.status.all_depends    = None
295
296     def myProgressCallback( self, x, y, f, from_cache ):
297         """Update any tty with the progress change"""
298         if os.isatty(sys.stdout.fileno()):
299             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
300             sys.stdout.flush()
301         else:
302             if x == 1:
303                 sys.stdout.write("Parsing .bb files, please wait...")
304                 sys.stdout.flush()
305             if x == y:
306                 sys.stdout.write("done.")
307                 sys.stdout.flush()
308
309     def interactiveMode( self ):
310         """Drop off into a shell"""
311         try:
312             from bb import shell
313         except ImportError, details:
314             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
315         else:
316             bb.data.update_data( self.configuration.data )
317             bb.data.expandKeys( self.configuration.data )
318             shell.start( self )
319             sys.exit( 0 )
320
321     def parseConfigurationFile( self, afile ):
322         try:
323             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
324
325             # Add the handlers we inherited by INHERIT
326             # we need to do this manually as it is not guranteed
327             # we will pick up these classes... as we only INHERIT
328             # on .inc and .bb files but not on .conf
329             data = bb.data.createCopy( self.configuration.data )
330             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
331             for inherit in inherits:
332                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
333
334             # FIXME: This assumes that we included at least one .inc file
335             for var in bb.data.keys(data):
336                 if bb.data.getVarFlag(var, 'handler', data):
337                     bb.event.register(var,bb.data.getVar(var, data))
338
339             bb.fetch.fetcher_init(self.configuration.data)
340
341         except IOError:
342             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
343         except bb.parse.ParseError, details:
344             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
345
346     def handleCollections( self, collections ):
347         """Handle collections"""
348         if collections:
349             collection_list = collections.split()
350             for c in collection_list:
351                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
352                 if regex == None:
353                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
354                     continue
355                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
356                 if priority == None:
357                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
358                     continue
359                 try:
360                     cre = re.compile(regex)
361                 except re.error:
362                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
363                     continue
364                 try:
365                     pri = int(priority)
366                     self.status.bbfile_config_priorities.append((cre, pri))
367                 except ValueError:
368                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
369
370     def buildSetVars(self):
371         """
372         Setup any variables needed before starting a build
373         """
374         if not bb.data.getVar("BUILDNAME", self.configuration.data):
375             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
376         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
377
378     def buildFile(self, buildfile):
379         """
380         Build the file matching regexp buildfile
381         """
382
383         bf = os.path.abspath(buildfile)
384         try:
385             os.stat(bf)
386         except OSError:
387             (filelist, masked) = self.collect_bbfiles()
388             regexp = re.compile(buildfile)
389             matches = []
390             for f in filelist:
391                 if regexp.search(f) and os.path.isfile(f):
392                     bf = f
393                     matches.append(f)
394             if len(matches) != 1:
395                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
396                 for f in matches:
397                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
398                 sys.exit(1)
399             bf = matches[0]                 
400
401         bbfile_data = bb.parse.handle(bf, self.configuration.data)
402
403         # Remove stamp for target if force mode active
404         if self.configuration.force:
405             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
406             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
407
408         item = bb.data.getVar('PN', bbfile_data, 1)
409         try:
410             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
411         except bb.build.EventException:
412             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
413
414         sys.exit(0)
415
416     def buildTargets(self, targets):
417         """
418         Attempt to build the targets specified
419         """
420
421         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
422         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
423
424         localdata = data.createCopy(self.configuration.data)
425         bb.data.update_data(localdata)
426         bb.data.expandKeys(localdata)
427
428         taskdata = bb.taskdata.TaskData(self.configuration.abort)
429
430         runlist = []
431         try:
432             for k in targets:
433                 taskdata.add_provider(localdata, self.status, k)
434                 runlist.append([k, "do_%s" % self.configuration.cmd])
435             taskdata.add_unresolved(localdata, self.status)
436         except bb.providers.NoProvider:
437             sys.exit(1)
438
439         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
440         rq.prepare_runqueue()
441         try:
442             failures = rq.execute_runqueue()
443         except runqueue.TaskFailure, fnids:
444             for fnid in fnids:
445                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
446             sys.exit(1)
447         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
448
449         sys.exit(0)
450
451     def updateCache(self):
452         # Import Psyco if available and not disabled
453         if not self.configuration.disable_psyco:
454             try:
455                 import psyco
456             except ImportError:
457                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
458             else:
459                 psyco.bind( self.parse_bbfiles )
460         else:
461             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
462
463         self.status = bb.cache.CacheData()
464
465         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
466         self.status.ignored_dependencies = Set( ignore.split() )
467
468         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
469
470         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
471         (filelist, masked) = self.collect_bbfiles()
472         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
473         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
474
475         self.buildDepgraph()
476
477     def cook(self):
478         """
479         We are building stuff here. We do the building
480         from here. By default we try to execute task
481         build.
482         """
483
484         if self.configuration.show_environment:
485             self.showEnvironment()
486             sys.exit( 0 )
487
488         self.buildSetVars()
489
490         if self.configuration.interactive:
491             self.interactiveMode()
492
493         if self.configuration.buildfile is not None:
494             return self.buildFile(self.configuration.buildfile)
495
496         # initialise the parsing status now we know we will need deps
497         self.updateCache()
498
499         if self.configuration.parse_only:
500             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
501             return 0
502
503         pkgs_to_build = self.configuration.pkgs_to_build
504
505         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
506         if bbpkgs:
507             pkgs_to_build.extend(bbpkgs.split())
508         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
509                              and not self.configuration.show_environment:
510                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
511                 print "for usage information."
512                 sys.exit(0)
513
514         try:
515             if self.configuration.show_versions:
516                 self.showVersions()
517                 sys.exit( 0 )
518             if 'world' in pkgs_to_build:
519                 self.buildWorldTargetList()
520                 pkgs_to_build.remove('world')
521                 for t in self.status.world_target:
522                     pkgs_to_build.append(t)
523
524             if self.configuration.dot_graph:
525                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
526                 sys.exit( 0 )
527
528             return self.buildTargets(pkgs_to_build)
529
530         except KeyboardInterrupt:
531             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
532             sys.exit(1)
533
534     def get_bbfiles( self, path = os.getcwd() ):
535         """Get list of default .bb files by reading out the current directory"""
536         contents = os.listdir(path)
537         bbfiles = []
538         for f in contents:
539             (root, ext) = os.path.splitext(f)
540             if ext == ".bb":
541                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
542         return bbfiles
543
544     def find_bbfiles( self, path ):
545         """Find all the .bb files in a directory"""
546         from os.path import join
547
548         found = []
549         for dir, dirs, files in os.walk(path):
550             for ignored in ('SCCS', 'CVS', '.svn'):
551                 if ignored in dirs:
552                     dirs.remove(ignored)
553             found += [join(dir,f) for f in files if f.endswith('.bb')]
554
555         return found
556
557     def collect_bbfiles( self ):
558         """Collect all available .bb build files"""
559         parsed, cached, skipped, masked = 0, 0, 0, 0
560         self.bb_cache = bb.cache.init(self)
561
562         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
563         data.setVar("BBFILES", " ".join(files), self.configuration.data)
564
565         if not len(files):
566             files = self.get_bbfiles()
567
568         if not len(files):
569             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
570
571         newfiles = []
572         for f in files:
573             if os.path.isdir(f):
574                 dirfiles = self.find_bbfiles(f)
575                 if dirfiles:
576                     newfiles += dirfiles
577                     continue
578             newfiles += glob.glob(f) or [ f ]
579
580         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
581
582         if not bbmask:
583             return (newfiles, 0)
584
585         try:
586             bbmask_compiled = re.compile(bbmask)
587         except sre_constants.error:
588             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
589
590         finalfiles = []
591         for i in xrange( len( newfiles ) ):
592             f = newfiles[i]
593             if bbmask and bbmask_compiled.search(f):
594                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
595                 masked += 1
596                 continue
597             finalfiles.append(f)
598
599         return (finalfiles, masked)
600
601     def parse_bbfiles(self, filelist, masked, progressCallback = None):
602         parsed, cached, skipped, error = 0, 0, 0, 0
603         for i in xrange( len( filelist ) ):
604             f = filelist[i]
605
606             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
607
608             # read a file's metadata
609             try:
610                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
611                 if skip:
612                     skipped += 1
613                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
614                     self.bb_cache.skip(f)
615                     continue
616                 elif fromCache: cached += 1
617                 else: parsed += 1
618                 deps = None
619
620                 # Disabled by RP as was no longer functional
621                 # allow metadata files to add items to BBFILES
622                 #data.update_data(self.pkgdata[f])
623                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
624                 #if addbbfiles:
625                 #    for aof in addbbfiles.split():
626                 #        if not files.count(aof):
627                 #            if not os.path.isabs(aof):
628                 #                aof = os.path.join(os.path.dirname(f),aof)
629                 #            files.append(aof)
630
631                 self.bb_cache.handle_data(f, self.status)
632
633                 # now inform the caller
634                 if progressCallback is not None:
635                     progressCallback( i + 1, len( filelist ), f, fromCache )
636
637             except IOError, e:
638                 self.bb_cache.remove(f)
639                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
640                 pass
641             except KeyboardInterrupt:
642                 self.bb_cache.sync()
643                 raise
644             except Exception, e:
645                 error += 1
646                 self.bb_cache.remove(f)
647                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
648             except:
649                 self.bb_cache.remove(f)
650                 raise
651
652         if progressCallback is not None:
653             print "\r" # need newline after Handling Bitbake files message
654             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
655
656         self.bb_cache.sync()
657
658         if error > 0:
659             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")