cooker.py: Clean up pkgs_to_build handling
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify it under
13 # the terms of the GNU General Public License as published by the Free Software
14 # Foundation; either version 2 of the License, or (at your option) any later
15 # version.
16 #
17 # This program is distributed in the hope that it will be useful, but WITHOUT
18 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
19 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along with
22 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
23 # Place, Suite 330, Boston, MA 02111-1307 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBStatistics
35 #============================================================================#
36 class BBStatistics:
37     """
38     Manage build statistics for one run
39     """
40     def __init__(self ):
41         self.attempt = 0
42         self.success = 0
43         self.fail = 0
44         self.deps = 0
45
46     def show( self ):
47         print "Build statistics:"
48         print "  Attempted builds: %d" % self.attempt
49         if self.fail:
50             print "  Failed builds: %d" % self.fail
51         if self.deps:
52             print "  Dependencies not satisfied: %d" % self.deps
53         if self.fail or self.deps: return 1
54         else: return 0
55
56 #============================================================================#
57 # BBCooker
58 #============================================================================#
59 class BBCooker:
60     """
61     Manages one bitbake build run
62     """
63
64     Statistics = BBStatistics           # make it visible from the shell
65
66     def __init__( self ):
67         self.build_cache_fail = []
68         self.build_cache = []
69         self.stats = BBStatistics()
70         self.status = None
71
72         self.cache = None
73         self.bb_cache = None
74
75     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
76         """
77         Build one task of a package, optionally build following task depends
78         """
79         bb.event.fire(bb.event.PkgStarted(item, the_data))
80         try:
81             self.stats.attempt += 1
82             if self.configuration.force:
83                 bb.data.setVarFlag('do_%s' % task, 'force', 1, the_data)
84             if not build_depends:
85                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
86             if not self.configuration.dry_run:
87                 bb.build.exec_task('do_%s' % task, the_data)
88             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
89             self.build_cache.append(fn)
90             return True
91         except bb.build.FuncFailed:
92             self.stats.fail += 1
93             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
94             bb.event.fire(bb.event.PkgFailed(item, the_data))
95             self.build_cache_fail.append(fn)
96             raise
97         except bb.build.EventException, e:
98             self.stats.fail += 1
99             event = e.args[1]
100             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
101             bb.event.fire(bb.event.PkgFailed(item, the_data))
102             self.build_cache_fail.append(fn)
103             raise
104
105     def tryBuild( self, fn, build_depends):
106         """
107         Build a provider and its dependencies. 
108         build_depends is a list of previous build dependencies (not runtime)
109         If build_depends is empty, we're dealing with a runtime depends
110         """
111
112         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
113
114         item = self.status.pkg_fn[fn]
115
116         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data) and not self.configuration.force:
117             self.build_cache.append(fn)
118             return True
119
120         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
121
122     def showVersions( self ):
123         pkg_pn = self.status.pkg_pn
124         preferred_versions = {}
125         latest_versions = {}
126
127         # Sort by priority
128         for pn in pkg_pn.keys():
129             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
130             preferred_versions[pn] = (pref_ver, pref_file)
131             latest_versions[pn] = (last_ver, last_file)
132
133         pkg_list = pkg_pn.keys()
134         pkg_list.sort()
135
136         for p in pkg_list:
137             pref = preferred_versions[p]
138             latest = latest_versions[p]
139
140             if pref != latest:
141                 prefstr = pref[0][0] + "-" + pref[0][1]
142             else:
143                 prefstr = ""
144
145             print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
146                                         prefstr)
147
148
149     def showEnvironment( self ):
150         """Show the outer or per-package environment"""
151         if self.configuration.buildfile:
152             self.cb = None
153             self.bb_cache = bb.cache.init(self)
154             try:
155                 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
156             except IOError, e:
157                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
158             except Exception, e:
159                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
160         # emit variables and shell functions
161         try:
162             data.update_data( self.configuration.data )
163             data.emit_env(sys.__stdout__, self.configuration.data, True)
164         except Exception, e:
165             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
166         # emit the metadata which isnt valid shell
167         data.expandKeys( self.configuration.data )      
168         for e in self.configuration.data.keys():
169             if data.getVarFlag( e, 'python', self.configuration.data ):
170                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
171
172     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
173         """
174         Generate a task dependency graph. 
175
176         pkgs_to_build A list of packages that needs to be built
177         ignore_deps   A list of names where processing of dependencies
178                       should be stopped. e.g. dependencies that get
179         """
180
181         for dep in ignore_deps:
182             self.status.ignored_dependencies.add(dep)
183
184         localdata = data.createCopy(self.configuration.data)
185         bb.data.update_data(localdata)
186         bb.data.expandKeys(localdata)
187         taskdata = bb.taskdata.TaskData(self.configuration.abort)
188
189         runlist = []
190         try:
191             for k in pkgs_to_build:
192                 taskdata.add_provider(localdata, self.status, k)
193                 runlist.append([k, "do_%s" % self.configuration.cmd])
194             taskdata.add_unresolved(localdata, self.status)
195         except bb.providers.NoProvider:
196             sys.exit(1)
197         rq = bb.runqueue.RunQueue()
198         rq.prepare_runqueue(self.configuration.data, self.status, taskdata, runlist)
199
200         seen_fnids = []  
201         depends_file = file('depends.dot', 'w' )
202         tdepends_file = file('task-depends.dot', 'w' )
203         print >> depends_file, "digraph depends {"
204         print >> tdepends_file, "digraph depends {"
205         rq.prio_map.reverse()
206         for task1 in range(len(rq.runq_fnid)):
207             task = rq.prio_map[task1]
208             taskname = rq.runq_task[task]
209             fnid = rq.runq_fnid[task]
210             fn = taskdata.fn_index[fnid]
211             pn = self.status.pkg_fn[fn]
212             version  = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
213             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
214             for dep in rq.runq_depends[task]:
215                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
216                 deppn = self.status.pkg_fn[depfn]
217                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
218             if fnid not in seen_fnids:
219                 seen_fnids.append(fnid)
220                 packages = []
221                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
222                 for depend in self.status.deps[fn]:
223                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
224                 rdepends = self.status.rundeps[fn]
225                 for package in rdepends:
226                     for rdepend in rdepends[package]:
227                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
228                     packages.append(package)
229                 rrecs = self.status.runrecs[fn]
230                 for package in rrecs:
231                     for rdepend in rrecs[package]:
232                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
233                     if not package in packages:
234                         packages.append(package)
235                 for package in packages:
236                     if package != pn:
237                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
238                         for depend in self.status.deps[fn]:
239                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
240                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
241                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
242                 #for rdep in taskdata.rdepids[fnid]:
243                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
244                 #for dep in taskdata.depids[fnid]:
245                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
246         print >> depends_file,  "}"
247         print >> tdepends_file,  "}"
248         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
249         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
250
251     def buildDepgraph( self ):
252         all_depends = self.status.all_depends
253         pn_provides = self.status.pn_provides
254
255         localdata = data.createCopy(self.configuration.data)
256         bb.data.update_data(localdata)
257         bb.data.expandKeys(localdata)
258
259         def calc_bbfile_priority(filename):
260             for (regex, pri) in self.status.bbfile_config_priorities:
261                 if regex.match(filename):
262                     return pri
263             return 0
264
265         # Handle PREFERRED_PROVIDERS
266         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
267             (providee, provider) = p.split(':')
268             if providee in self.status.preferred and self.status.preferred[providee] != provider:
269                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
270             self.status.preferred[providee] = provider
271
272         # Calculate priorities for each file
273         for p in self.status.pkg_fn.keys():
274             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
275
276     def buildWorldTargetList(self):
277         """
278          Build package list for "bitbake world"
279         """
280         all_depends = self.status.all_depends
281         pn_provides = self.status.pn_provides
282         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
283         for f in self.status.possible_world:
284             terminal = True
285             pn = self.status.pkg_fn[f]
286
287             for p in pn_provides[pn]:
288                 if p.startswith('virtual/'):
289                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
290                     terminal = False
291                     break
292                 for pf in self.status.providers[p]:
293                     if self.status.pkg_fn[pf] != pn:
294                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
295                         terminal = False
296                         break
297             if terminal:
298                 self.status.world_target.add(pn)
299
300             # drop reference count now
301             self.status.possible_world = None
302             self.status.all_depends    = None
303
304     def myProgressCallback( self, x, y, f, from_cache ):
305         """Update any tty with the progress change"""
306         if os.isatty(sys.stdout.fileno()):
307             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
308             sys.stdout.flush()
309         else:
310             if x == 1:
311                 sys.stdout.write("Parsing .bb files, please wait...")
312                 sys.stdout.flush()
313             if x == y:
314                 sys.stdout.write("done.")
315                 sys.stdout.flush()
316
317     def interactiveMode( self ):
318         """Drop off into a shell"""
319         try:
320             from bb import shell
321         except ImportError, details:
322             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
323         else:
324             bb.data.update_data( self.configuration.data )
325             bb.data.expandKeys( self.configuration.data )
326             shell.start( self )
327             sys.exit( 0 )
328
329     def parseConfigurationFile( self, afile ):
330         try:
331             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
332
333             # Add the handlers we inherited by INHERIT
334             # we need to do this manually as it is not guranteed
335             # we will pick up these classes... as we only INHERIT
336             # on .inc and .bb files but not on .conf
337             data = bb.data.createCopy( self.configuration.data )
338             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
339             for inherit in inherits:
340                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
341
342             # FIXME: This assumes that we included at least one .inc file
343             for var in bb.data.keys(data):
344                 if bb.data.getVarFlag(var, 'handler', data):
345                     bb.event.register(var,bb.data.getVar(var, data))
346
347         except IOError:
348             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
349         except bb.parse.ParseError, details:
350             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
351
352     def handleCollections( self, collections ):
353         """Handle collections"""
354         if collections:
355             collection_list = collections.split()
356             for c in collection_list:
357                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
358                 if regex == None:
359                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
360                     continue
361                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
362                 if priority == None:
363                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
364                     continue
365                 try:
366                     cre = re.compile(regex)
367                 except re.error:
368                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
369                     continue
370                 try:
371                     pri = int(priority)
372                     self.status.bbfile_config_priorities.append((cre, pri))
373                 except ValueError:
374                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
375
376
377     def cook(self, configuration):
378         """
379         We are building stuff here. We do the building
380         from here. By default we try to execute task
381         build.
382         """
383
384         self.configuration = configuration
385
386         if self.configuration.verbose:
387             bb.msg.set_verbose(True)
388
389         if self.configuration.debug:
390             bb.msg.set_debug_level(self.configuration.debug)
391         else:
392             bb.msg.set_debug_level(0)
393
394         if self.configuration.debug_domains:
395             bb.msg.set_debug_domains(self.configuration.debug_domains)
396
397         self.configuration.data = bb.data.init()
398
399         for f in self.configuration.file:
400             self.parseConfigurationFile( f )
401
402         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
403
404         if not self.configuration.cmd:
405             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data)
406
407         # For backwards compatibility - REMOVE ME
408         if not self.configuration.cmd:
409             self.configuration.cmd = "build"
410
411         #
412         # Special updated configuration we use for firing events
413         #
414         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
415         bb.data.update_data(self.configuration.event_data)
416
417         if self.configuration.show_environment:
418             self.showEnvironment()
419             sys.exit( 0 )
420
421         # inject custom variables
422         if not bb.data.getVar("BUILDNAME", self.configuration.data):
423             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
424         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
425
426         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
427
428         if self.configuration.interactive:
429             self.interactiveMode()
430
431         if self.configuration.buildfile is not None:
432             bf = os.path.abspath( self.configuration.buildfile )
433             try:
434                 os.stat(bf)
435             except OSError:
436                 (filelist, masked) = self.collect_bbfiles()
437                 regexp = re.compile(self.configuration.buildfile)
438                 matches = []
439                 for f in filelist:
440                     if regexp.search(f) and os.path.isfile(f):
441                         bf = f
442                         matches.append(f)
443                 if len(matches) != 1:
444                     bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
445                     for f in matches:
446                         bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
447                     sys.exit(1)
448                 bf = matches[0]             
449
450             bbfile_data = bb.parse.handle(bf, self.configuration.data)
451
452             item = bb.data.getVar('PN', bbfile_data, 1)
453             try:
454                 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
455             except bb.build.EventException:
456                 bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
457
458             sys.exit( self.stats.show() )
459
460         # initialise the parsing status now we know we will need deps
461         self.status = bb.cache.CacheData()
462
463         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
464         self.status.ignored_dependencies = Set( ignore.split() )
465
466         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
467
468         pkgs_to_build = self.configuration.pkgs_to_build
469
470         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
471         if bbpkgs:
472             pkgs_to_build.extend(bbpkgs.split())
473         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
474                              and not self.configuration.show_environment:
475                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
476                 print "for usage information."
477                 sys.exit(0)
478
479         # Import Psyco if available and not disabled
480         if not self.configuration.disable_psyco:
481             try:
482                 import psyco
483             except ImportError:
484                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
485             else:
486                 psyco.bind( self.parse_bbfiles )
487         else:
488             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
489
490         try:
491             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
492             (filelist, masked) = self.collect_bbfiles()
493             self.parse_bbfiles(filelist, masked, self.myProgressCallback)
494             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
495             print
496             if self.configuration.parse_only:
497                 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
498                 return
499
500
501             self.buildDepgraph()
502
503             if self.configuration.show_versions:
504                 self.showVersions()
505                 sys.exit( 0 )
506             if 'world' in pkgs_to_build:
507                 self.buildWorldTargetList()
508                 pkgs_to_build.remove('world')
509                 for t in self.status.world_target:
510                     pkgs_to_build.append(t)
511
512             if self.configuration.dot_graph:
513                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
514                 sys.exit( 0 )
515
516             bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
517
518             localdata = data.createCopy(self.configuration.data)
519             bb.data.update_data(localdata)
520             bb.data.expandKeys(localdata)
521
522             taskdata = bb.taskdata.TaskData(self.configuration.abort)
523
524             runlist = []
525             try:
526                 for k in pkgs_to_build:
527                     taskdata.add_provider(localdata, self.status, k)
528                     runlist.append([k, "do_%s" % self.configuration.cmd])
529                 taskdata.add_unresolved(localdata, self.status)
530             except bb.providers.NoProvider:
531                 sys.exit(1)
532
533             rq = bb.runqueue.RunQueue()
534             rq.prepare_runqueue(self.configuration.data, self.status, taskdata, runlist)
535             try:
536                 failures = rq.execute_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
537             except runqueue.TaskFailure, fnids:
538                 for fnid in fnids:
539                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
540                 sys.exit(1)
541             bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
542
543             sys.exit( self.stats.show() )
544
545         except KeyboardInterrupt:
546             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
547             sys.exit(1)
548
549     def get_bbfiles( self, path = os.getcwd() ):
550         """Get list of default .bb files by reading out the current directory"""
551         contents = os.listdir(path)
552         bbfiles = []
553         for f in contents:
554             (root, ext) = os.path.splitext(f)
555             if ext == ".bb":
556                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
557         return bbfiles
558
559     def find_bbfiles( self, path ):
560         """Find all the .bb files in a directory (uses find)"""
561         findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
562         try:
563             finddata = os.popen(findcmd)
564         except OSError:
565             return []
566         return finddata.readlines()
567
568     def collect_bbfiles( self ):
569         """Collect all available .bb build files"""
570         parsed, cached, skipped, masked = 0, 0, 0, 0
571         self.bb_cache = bb.cache.init(self)
572
573         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
574         data.setVar("BBFILES", " ".join(files), self.configuration.data)
575
576         if not len(files):
577             files = self.get_bbfiles()
578
579         if not len(files):
580             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
581
582         newfiles = []
583         for f in files:
584             if os.path.isdir(f):
585                 dirfiles = self.find_bbfiles(f)
586                 if dirfiles:
587                     newfiles += dirfiles
588                     continue
589             newfiles += glob.glob(f) or [ f ]
590
591         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
592
593         if not bbmask:
594             return (newfiles, 0)
595
596         try:
597             bbmask_compiled = re.compile(bbmask)
598         except sre_constants.error:
599             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
600
601         finalfiles = []
602         for i in xrange( len( newfiles ) ):
603             f = newfiles[i]
604             if bbmask and bbmask_compiled.search(f):
605                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
606                 masked += 1
607                 continue
608             finalfiles.append(f)
609
610         return (finalfiles, masked)
611
612     def parse_bbfiles(self, filelist, masked, progressCallback = None):
613         parsed, cached, skipped = 0, 0, 0
614         for i in xrange( len( filelist ) ):
615             f = filelist[i]
616
617             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
618
619             # read a file's metadata
620             try:
621                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
622                 if skip:
623                     skipped += 1
624                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
625                     self.bb_cache.skip(f)
626                     continue
627                 elif fromCache: cached += 1
628                 else: parsed += 1
629                 deps = None
630
631                 # Disabled by RP as was no longer functional
632                 # allow metadata files to add items to BBFILES
633                 #data.update_data(self.pkgdata[f])
634                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
635                 #if addbbfiles:
636                 #    for aof in addbbfiles.split():
637                 #        if not files.count(aof):
638                 #            if not os.path.isabs(aof):
639                 #                aof = os.path.join(os.path.dirname(f),aof)
640                 #            files.append(aof)
641
642                 self.bb_cache.handle_data(f, self.status)
643
644                 # now inform the caller
645                 if progressCallback is not None:
646                     progressCallback( i + 1, len( filelist ), f, fromCache )
647
648             except IOError, e:
649                 self.bb_cache.remove(f)
650                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
651                 pass
652             except KeyboardInterrupt:
653                 self.bb_cache.sync()
654                 raise
655             except Exception, e:
656                 self.bb_cache.remove(f)
657                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
658             except:
659                 self.bb_cache.remove(f)
660                 raise
661
662         if progressCallback is not None:
663             print "\r" # need newline after Handling Bitbake files message
664             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
665
666         self.bb_cache.sync()