cooker.py Add support for BB_NICE_LEVEL option (#1627) [backport]
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76         #
77         # TOSTOP must not be set or our children will hang when they output
78         #
79         fd = sys.stdout.fileno()
80         if os.isatty(fd):
81             import termios
82             tcattr = termios.tcgetattr(fd)
83             if tcattr[3] & termios.TOSTOP:
84                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
85                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
86                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
87
88         # Change nice level if we're asked to
89         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
90         if nice:
91             curnice = os.nice(0)
92             nice = int(nice) - curnice
93             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
94  
95
96     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
97         """
98         Build one task of a package, optionally build following task depends
99         """
100         bb.event.fire(bb.event.PkgStarted(item, the_data))
101         try:
102             if not build_depends:
103                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
104             if not self.configuration.dry_run:
105                 bb.build.exec_task('do_%s' % task, the_data)
106             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
107             return True
108         except bb.build.FuncFailed:
109             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
110             bb.event.fire(bb.event.PkgFailed(item, the_data))
111             raise
112         except bb.build.EventException, e:
113             event = e.args[1]
114             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
115             bb.event.fire(bb.event.PkgFailed(item, the_data))
116             raise
117
118     def tryBuild( self, fn, build_depends):
119         """
120         Build a provider and its dependencies. 
121         build_depends is a list of previous build dependencies (not runtime)
122         If build_depends is empty, we're dealing with a runtime depends
123         """
124
125         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
126
127         item = self.status.pkg_fn[fn]
128
129         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
130             return True
131
132         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
133
134     def showVersions(self):
135         pkg_pn = self.status.pkg_pn
136         preferred_versions = {}
137         latest_versions = {}
138
139         # Sort by priority
140         for pn in pkg_pn.keys():
141             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
142             preferred_versions[pn] = (pref_ver, pref_file)
143             latest_versions[pn] = (last_ver, last_file)
144
145         pkg_list = pkg_pn.keys()
146         pkg_list.sort()
147
148         for p in pkg_list:
149             pref = preferred_versions[p]
150             latest = latest_versions[p]
151
152             if pref != latest:
153                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
154             else:
155                 prefstr = ""
156
157             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
158                                         prefstr)
159
160
161     def showEnvironment( self ):
162         """Show the outer or per-package environment"""
163         if self.configuration.buildfile:
164             self.cb = None
165             self.bb_cache = bb.cache.init(self)
166             bf = self.matchFile(self.configuration.buildfile)
167             try:
168                 self.configuration.data = self.bb_cache.loadDataFull(bf, self.configuration.data)
169             except IOError, e:
170                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (bf, e))
171             except Exception, e:
172                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
173         # emit variables and shell functions
174         try:
175             data.update_data( self.configuration.data )
176             data.emit_env(sys.__stdout__, self.configuration.data, True)
177         except Exception, e:
178             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
179         # emit the metadata which isnt valid shell
180         data.expandKeys( self.configuration.data )      
181         for e in self.configuration.data.keys():
182             if data.getVarFlag( e, 'python', self.configuration.data ):
183                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
184
185     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
186         """
187         Generate a task dependency graph. 
188
189         pkgs_to_build A list of packages that needs to be built
190         ignore_deps   A list of names where processing of dependencies
191                       should be stopped. e.g. dependencies that get
192         """
193
194         for dep in ignore_deps:
195             self.status.ignored_dependencies.add(dep)
196
197         localdata = data.createCopy(self.configuration.data)
198         bb.data.update_data(localdata)
199         bb.data.expandKeys(localdata)
200         taskdata = bb.taskdata.TaskData(self.configuration.abort)
201
202         runlist = []
203         try:
204             for k in pkgs_to_build:
205                 taskdata.add_provider(localdata, self.status, k)
206                 runlist.append([k, "do_%s" % self.configuration.cmd])
207             taskdata.add_unresolved(localdata, self.status)
208         except bb.providers.NoProvider:
209             sys.exit(1)
210         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
211         rq.prepare_runqueue()
212
213         seen_fnids = []  
214         depends_file = file('depends.dot', 'w' )
215         tdepends_file = file('task-depends.dot', 'w' )
216         print >> depends_file, "digraph depends {"
217         print >> tdepends_file, "digraph depends {"
218         rq.prio_map.reverse()
219         for task1 in range(len(rq.runq_fnid)):
220             task = rq.prio_map[task1]
221             taskname = rq.runq_task[task]
222             fnid = rq.runq_fnid[task]
223             fn = taskdata.fn_index[fnid]
224             pn = self.status.pkg_fn[fn]
225             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
226             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
227             for dep in rq.runq_depends[task]:
228                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
229                 deppn = self.status.pkg_fn[depfn]
230                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
231             if fnid not in seen_fnids:
232                 seen_fnids.append(fnid)
233                 packages = []
234                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
235                 for depend in self.status.deps[fn]:
236                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
237                 rdepends = self.status.rundeps[fn]
238                 for package in rdepends:
239                     for rdepend in rdepends[package]:
240                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
241                     packages.append(package)
242                 rrecs = self.status.runrecs[fn]
243                 for package in rrecs:
244                     for rdepend in rrecs[package]:
245                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
246                     if not package in packages:
247                         packages.append(package)
248                 for package in packages:
249                     if package != pn:
250                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
251                         for depend in self.status.deps[fn]:
252                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
253                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
254                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
255                 #for rdep in taskdata.rdepids[fnid]:
256                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
257                 #for dep in taskdata.depids[fnid]:
258                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
259         print >> depends_file,  "}"
260         print >> tdepends_file,  "}"
261         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
262         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
263
264     def buildDepgraph( self ):
265         all_depends = self.status.all_depends
266         pn_provides = self.status.pn_provides
267
268         localdata = data.createCopy(self.configuration.data)
269         bb.data.update_data(localdata)
270         bb.data.expandKeys(localdata)
271
272         def calc_bbfile_priority(filename):
273             for (regex, pri) in self.status.bbfile_config_priorities:
274                 if regex.match(filename):
275                     return pri
276             return 0
277
278         # Handle PREFERRED_PROVIDERS
279         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
280             (providee, provider) = p.split(':')
281             if providee in self.status.preferred and self.status.preferred[providee] != provider:
282                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
283             self.status.preferred[providee] = provider
284
285         # Calculate priorities for each file
286         for p in self.status.pkg_fn.keys():
287             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
288
289     def buildWorldTargetList(self):
290         """
291          Build package list for "bitbake world"
292         """
293         all_depends = self.status.all_depends
294         pn_provides = self.status.pn_provides
295         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
296         for f in self.status.possible_world:
297             terminal = True
298             pn = self.status.pkg_fn[f]
299
300             for p in pn_provides[pn]:
301                 if p.startswith('virtual/'):
302                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
303                     terminal = False
304                     break
305                 for pf in self.status.providers[p]:
306                     if self.status.pkg_fn[pf] != pn:
307                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
308                         terminal = False
309                         break
310             if terminal:
311                 self.status.world_target.add(pn)
312
313             # drop reference count now
314             self.status.possible_world = None
315             self.status.all_depends    = None
316
317     def myProgressCallback( self, x, y, f, from_cache ):
318         """Update any tty with the progress change"""
319         if os.isatty(sys.stdout.fileno()):
320             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
321             sys.stdout.flush()
322         else:
323             if x == 1:
324                 sys.stdout.write("Parsing .bb files, please wait...")
325                 sys.stdout.flush()
326             if x == y:
327                 sys.stdout.write("done.")
328                 sys.stdout.flush()
329
330     def interactiveMode( self ):
331         """Drop off into a shell"""
332         try:
333             from bb import shell
334         except ImportError, details:
335             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
336         else:
337             bb.data.update_data( self.configuration.data )
338             bb.data.expandKeys( self.configuration.data )
339             shell.start( self )
340             sys.exit( 0 )
341
342     def parseConfigurationFile( self, afile ):
343         try:
344             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
345
346             # Add the handlers we inherited by INHERIT
347             # we need to do this manually as it is not guranteed
348             # we will pick up these classes... as we only INHERIT
349             # on .inc and .bb files but not on .conf
350             data = bb.data.createCopy( self.configuration.data )
351             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
352             for inherit in inherits:
353                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
354
355             # FIXME: This assumes that we included at least one .inc file
356             for var in bb.data.keys(data):
357                 if bb.data.getVarFlag(var, 'handler', data):
358                     bb.event.register(var,bb.data.getVar(var, data))
359
360             bb.fetch.fetcher_init(self.configuration.data)
361
362             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
363
364         except IOError:
365             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
366         except bb.parse.ParseError, details:
367             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
368
369     def handleCollections( self, collections ):
370         """Handle collections"""
371         if collections:
372             collection_list = collections.split()
373             for c in collection_list:
374                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
375                 if regex == None:
376                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
377                     continue
378                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
379                 if priority == None:
380                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
381                     continue
382                 try:
383                     cre = re.compile(regex)
384                 except re.error:
385                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
386                     continue
387                 try:
388                     pri = int(priority)
389                     self.status.bbfile_config_priorities.append((cre, pri))
390                 except ValueError:
391                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
392
393     def buildSetVars(self):
394         """
395         Setup any variables needed before starting a build
396         """
397         if not bb.data.getVar("BUILDNAME", self.configuration.data):
398             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
399         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
400
401     def matchFile(self, buildfile):
402         """
403         Convert the fragment buildfile into a real file
404         Error if there are too many matches
405         """
406         bf = os.path.abspath(buildfile)
407         try:
408             os.stat(bf)
409             return bf
410         except OSError:
411             (filelist, masked) = self.collect_bbfiles()
412             regexp = re.compile(buildfile)
413             matches = []
414             for f in filelist:
415                 if regexp.search(f) and os.path.isfile(f):
416                     bf = f
417                     matches.append(f)
418             if len(matches) != 1:
419                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
420                 for f in matches:
421                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
422                 sys.exit(1)
423             return matches[0]               
424
425     def buildFile(self, buildfile):
426         """
427         Build the file matching regexp buildfile
428         """
429
430         bf = self.matchFile(buildfile)
431
432         bbfile_data = bb.parse.handle(bf, self.configuration.data)
433
434         # Remove stamp for target if force mode active
435         if self.configuration.force:
436             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
437             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
438
439         item = bb.data.getVar('PN', bbfile_data, 1)
440         try:
441             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
442         except bb.build.EventException:
443             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
444
445         sys.exit(0)
446
447     def buildTargets(self, targets):
448         """
449         Attempt to build the targets specified
450         """
451
452         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
453         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
454
455         localdata = data.createCopy(self.configuration.data)
456         bb.data.update_data(localdata)
457         bb.data.expandKeys(localdata)
458
459         taskdata = bb.taskdata.TaskData(self.configuration.abort)
460
461         runlist = []
462         try:
463             for k in targets:
464                 taskdata.add_provider(localdata, self.status, k)
465                 runlist.append([k, "do_%s" % self.configuration.cmd])
466             taskdata.add_unresolved(localdata, self.status)
467         except bb.providers.NoProvider:
468             sys.exit(1)
469
470         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
471         rq.prepare_runqueue()
472         try:
473             failures = rq.execute_runqueue()
474         except runqueue.TaskFailure, fnids:
475             for fnid in fnids:
476                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
477             sys.exit(1)
478         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
479
480         sys.exit(0)
481
482     def updateCache(self):
483         # Import Psyco if available and not disabled
484         import platform
485         if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
486             if not self.configuration.disable_psyco:
487                 try:
488                     import psyco
489                 except ImportError:
490                     bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
491                 else:
492                     psyco.bind( self.parse_bbfiles )
493             else:
494                 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
495
496         self.status = bb.cache.CacheData()
497
498         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
499         self.status.ignored_dependencies = Set( ignore.split() )
500
501         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
502
503         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
504         (filelist, masked) = self.collect_bbfiles()
505         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
506         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
507
508         self.buildDepgraph()
509
510     def cook(self):
511         """
512         We are building stuff here. We do the building
513         from here. By default we try to execute task
514         build.
515         """
516
517         if self.configuration.show_environment:
518             self.showEnvironment()
519             sys.exit( 0 )
520
521         self.buildSetVars()
522
523         if self.configuration.interactive:
524             self.interactiveMode()
525
526         if self.configuration.buildfile is not None:
527             return self.buildFile(self.configuration.buildfile)
528
529         # initialise the parsing status now we know we will need deps
530         self.updateCache()
531
532         if self.configuration.parse_only:
533             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
534             return 0
535
536         pkgs_to_build = self.configuration.pkgs_to_build
537
538         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
539         if bbpkgs:
540             pkgs_to_build.extend(bbpkgs.split())
541         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
542                              and not self.configuration.show_environment:
543                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
544                 print "for usage information."
545                 sys.exit(0)
546
547         try:
548             if self.configuration.show_versions:
549                 self.showVersions()
550                 sys.exit( 0 )
551             if 'world' in pkgs_to_build:
552                 self.buildWorldTargetList()
553                 pkgs_to_build.remove('world')
554                 for t in self.status.world_target:
555                     pkgs_to_build.append(t)
556
557             if self.configuration.dot_graph:
558                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
559                 sys.exit( 0 )
560
561             return self.buildTargets(pkgs_to_build)
562
563         except KeyboardInterrupt:
564             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
565             sys.exit(1)
566
567     def get_bbfiles( self, path = os.getcwd() ):
568         """Get list of default .bb files by reading out the current directory"""
569         contents = os.listdir(path)
570         bbfiles = []
571         for f in contents:
572             (root, ext) = os.path.splitext(f)
573             if ext == ".bb":
574                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
575         return bbfiles
576
577     def find_bbfiles( self, path ):
578         """Find all the .bb files in a directory"""
579         from os.path import join
580
581         found = []
582         for dir, dirs, files in os.walk(path):
583             for ignored in ('SCCS', 'CVS', '.svn'):
584                 if ignored in dirs:
585                     dirs.remove(ignored)
586             found += [join(dir,f) for f in files if f.endswith('.bb')]
587
588         return found
589
590     def collect_bbfiles( self ):
591         """Collect all available .bb build files"""
592         parsed, cached, skipped, masked = 0, 0, 0, 0
593         self.bb_cache = bb.cache.init(self)
594
595         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
596         data.setVar("BBFILES", " ".join(files), self.configuration.data)
597
598         if not len(files):
599             files = self.get_bbfiles()
600
601         if not len(files):
602             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
603
604         newfiles = []
605         for f in files:
606             if os.path.isdir(f):
607                 dirfiles = self.find_bbfiles(f)
608                 if dirfiles:
609                     newfiles += dirfiles
610                     continue
611             newfiles += glob.glob(f) or [ f ]
612
613         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
614
615         if not bbmask:
616             return (newfiles, 0)
617
618         try:
619             bbmask_compiled = re.compile(bbmask)
620         except sre_constants.error:
621             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
622
623         finalfiles = []
624         for i in xrange( len( newfiles ) ):
625             f = newfiles[i]
626             if bbmask and bbmask_compiled.search(f):
627                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
628                 masked += 1
629                 continue
630             finalfiles.append(f)
631
632         return (finalfiles, masked)
633
634     def parse_bbfiles(self, filelist, masked, progressCallback = None):
635         parsed, cached, skipped, error = 0, 0, 0, 0
636         for i in xrange( len( filelist ) ):
637             f = filelist[i]
638
639             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
640
641             # read a file's metadata
642             try:
643                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
644                 if skip:
645                     skipped += 1
646                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
647                     self.bb_cache.skip(f)
648                     continue
649                 elif fromCache: cached += 1
650                 else: parsed += 1
651                 deps = None
652
653                 # Disabled by RP as was no longer functional
654                 # allow metadata files to add items to BBFILES
655                 #data.update_data(self.pkgdata[f])
656                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
657                 #if addbbfiles:
658                 #    for aof in addbbfiles.split():
659                 #        if not files.count(aof):
660                 #            if not os.path.isabs(aof):
661                 #                aof = os.path.join(os.path.dirname(f),aof)
662                 #            files.append(aof)
663
664                 self.bb_cache.handle_data(f, self.status)
665
666                 # now inform the caller
667                 if progressCallback is not None:
668                     progressCallback( i + 1, len( filelist ), f, fromCache )
669
670             except IOError, e:
671                 self.bb_cache.remove(f)
672                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
673                 pass
674             except KeyboardInterrupt:
675                 self.bb_cache.sync()
676                 raise
677             except Exception, e:
678                 error += 1
679                 self.bb_cache.remove(f)
680                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
681             except:
682                 self.bb_cache.remove(f)
683                 raise
684
685         if progressCallback is not None:
686             print "\r" # need newline after Handling Bitbake files message
687             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
688
689         self.bb_cache.sync()
690
691         if error > 0:
692             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")