lib/bb/cooker.py

   1 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 #
   5 # Copyright (C) 2003, 2004  Chris Larson
   6 # Copyright (C) 2003, 2004  Phil Blundell
   7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
   8 # Copyright (C) 2005        Holger Hans Peter Freyther
   9 # Copyright (C) 2005        ROAD GmbH
  10 # Copyright (C) 2006        Richard Purdie
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License version 2 as
  14 # published by the Free Software Foundation.
  15 #
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20 #
  21 # You should have received a copy of the GNU General Public License along
  22 # with this program; if not, write to the Free Software Foundation, Inc.,
  23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  24
  25 import sys, os, getopt, glob, copy, os.path, re, time
  26 import bb
  27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
  28 from sets import Set
  29 import itertools, sre_constants
  30
  31 parsespin = itertools.cycle( r'|/-\\' )
  32
  33 #============================================================================#
  34 # BBCooker
  35 #============================================================================#
  36 class BBCooker:
  37     """
  38     Manages one bitbake build run
  39     """
  40
  41     def __init__(self, configuration):
  42         self.status = None
  43
  44         self.cache = None
  45         self.bb_cache = None
  46
  47         self.configuration = configuration
  48
  49         if self.configuration.verbose:
  50             bb.msg.set_verbose(True)
  51
  52         if self.configuration.debug:
  53             bb.msg.set_debug_level(self.configuration.debug)
  54         else:
  55             bb.msg.set_debug_level(0)
  56
  57         if self.configuration.debug_domains:
  58             bb.msg.set_debug_domains(self.configuration.debug_domains)
  59
  60         self.configuration.data = bb.data.init()
  61
  62         for f in self.configuration.file:
  63             self.parseConfigurationFile( f )
  64
  65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
  66
  67         if not self.configuration.cmd:
  68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
  69
  70         #
  71         # Special updated configuration we use for firing events
  72         #
  73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
  74         bb.data.update_data(self.configuration.event_data)
  75
  76     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
  77         """
  78         Build one task of a package, optionally build following task depends
  79         """
  80         bb.event.fire(bb.event.PkgStarted(item, the_data))
  81         try:
  82             if not build_depends:
  83                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
  84             if not self.configuration.dry_run:
  85                 bb.build.exec_task('do_%s' % task, the_data)
  86             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
  87             return True
  88         except bb.build.FuncFailed:
  89             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
  90             bb.event.fire(bb.event.PkgFailed(item, the_data))
  91             raise
  92         except bb.build.EventException, e:
  93             event = e.args[1]
  94             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
  95             bb.event.fire(bb.event.PkgFailed(item, the_data))
  96             raise
  97
  98     def tryBuild( self, fn, build_depends):
  99         """
 100         Build a provider and its dependencies.
 101         build_depends is a list of previous build dependencies (not runtime)
 102         If build_depends is empty, we're dealing with a runtime depends
 103         """
 104
 105         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
 106
 107         item = self.status.pkg_fn[fn]
 108
 109         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
 110             return True
 111
 112         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
 113
 114     def showVersions(self):
 115         pkg_pn = self.status.pkg_pn
 116         preferred_versions = {}
 117         latest_versions = {}
 118
 119         # Sort by priority
 120         for pn in pkg_pn.keys():
 121             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
 122             preferred_versions[pn] = (pref_ver, pref_file)
 123             latest_versions[pn] = (last_ver, last_file)
 124
 125         pkg_list = pkg_pn.keys()
 126         pkg_list.sort()
 127
 128         for p in pkg_list:
 129             pref = preferred_versions[p]
 130             latest = latest_versions[p]
 131
 132             if pref != latest:
 133                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
 134             else:
 135                 prefstr = ""
 136
 137             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
 138                                         prefstr)
 139
 140
 141     def showEnvironment( self ):
 142         """Show the outer or per-package environment"""
 143         if self.configuration.buildfile:
 144             self.cb = None
 145             self.bb_cache = bb.cache.init(self)
 146             try:
 147                 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
 148             except IOError, e:
 149                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
 150             except Exception, e:
 151                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
 152         # emit variables and shell functions
 153         try:
 154             data.update_data( self.configuration.data )
 155             data.emit_env(sys.__stdout__, self.configuration.data, True)
 156         except Exception, e:
 157             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
 158         # emit the metadata which isnt valid shell
 159         data.expandKeys( self.configuration.data )
 160         for e in self.configuration.data.keys():
 161             if data.getVarFlag( e, 'python', self.configuration.data ):
 162                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
 163
 164     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
 165         """
 166         Generate a task dependency graph.
 167
 168         pkgs_to_build A list of packages that needs to be built
 169         ignore_deps   A list of names where processing of dependencies
 170                       should be stopped. e.g. dependencies that get
 171         """
 172
 173         for dep in ignore_deps:
 174             self.status.ignored_dependencies.add(dep)
 175
 176         localdata = data.createCopy(self.configuration.data)
 177         bb.data.update_data(localdata)
 178         bb.data.expandKeys(localdata)
 179         taskdata = bb.taskdata.TaskData(self.configuration.abort)
 180
 181         runlist = []
 182         try:
 183             for k in pkgs_to_build:
 184                 taskdata.add_provider(localdata, self.status, k)
 185                 runlist.append([k, "do_%s" % self.configuration.cmd])
 186             taskdata.add_unresolved(localdata, self.status)
 187         except bb.providers.NoProvider:
 188             sys.exit(1)
 189         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 190         rq.prepare_runqueue()
 191
 192         seen_fnids = []
 193         depends_file = file('depends.dot', 'w' )
 194         tdepends_file = file('task-depends.dot', 'w' )
 195         print >> depends_file, "digraph depends {"
 196         print >> tdepends_file, "digraph depends {"
 197         rq.prio_map.reverse()
 198         for task1 in range(len(rq.runq_fnid)):
 199             task = rq.prio_map[task1]
 200             taskname = rq.runq_task[task]
 201             fnid = rq.runq_fnid[task]
 202             fn = taskdata.fn_index[fnid]
 203             pn = self.status.pkg_fn[fn]
 204             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
 205             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
 206             for dep in rq.runq_depends[task]:
 207                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
 208                 deppn = self.status.pkg_fn[depfn]
 209                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
 210             if fnid not in seen_fnids:
 211                 seen_fnids.append(fnid)
 212                 packages = []
 213                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
 214                 for depend in self.status.deps[fn]:
 215                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
 216                 rdepends = self.status.rundeps[fn]
 217                 for package in rdepends:
 218                     for rdepend in rdepends[package]:
 219                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
 220                     packages.append(package)
 221                 rrecs = self.status.runrecs[fn]
 222                 for package in rrecs:
 223                     for rdepend in rrecs[package]:
 224                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
 225                     if not package in packages:
 226                         packages.append(package)
 227                 for package in packages:
 228                     if package != pn:
 229                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
 230                         for depend in self.status.deps[fn]:
 231                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
 232                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
 233                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
 234                 #for rdep in taskdata.rdepids[fnid]:
 235                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
 236                 #for dep in taskdata.depids[fnid]:
 237                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
 238         print >> depends_file,  "}"
 239         print >> tdepends_file,  "}"
 240         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
 241         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
 242
 243     def buildDepgraph( self ):
 244         all_depends = self.status.all_depends
 245         pn_provides = self.status.pn_provides
 246
 247         localdata = data.createCopy(self.configuration.data)
 248         bb.data.update_data(localdata)
 249         bb.data.expandKeys(localdata)
 250
 251         def calc_bbfile_priority(filename):
 252             for (regex, pri) in self.status.bbfile_config_priorities:
 253                 if regex.match(filename):
 254                     return pri
 255             return 0
 256
 257         # Handle PREFERRED_PROVIDERS
 258         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
 259             (providee, provider) = p.split(':')
 260             if providee in self.status.preferred and self.status.preferred[providee] != provider:
 261                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
 262             self.status.preferred[providee] = provider
 263
 264         # Calculate priorities for each file
 265         for p in self.status.pkg_fn.keys():
 266             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
 267
 268     def buildWorldTargetList(self):
 269         """
 270          Build package list for "bitbake world"
 271         """
 272         all_depends = self.status.all_depends
 273         pn_provides = self.status.pn_provides
 274         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
 275         for f in self.status.possible_world:
 276             terminal = True
 277             pn = self.status.pkg_fn[f]
 278
 279             for p in pn_provides[pn]:
 280                 if p.startswith('virtual/'):
 281                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
 282                     terminal = False
 283                     break
 284                 for pf in self.status.providers[p]:
 285                     if self.status.pkg_fn[pf] != pn:
 286                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
 287                         terminal = False
 288                         break
 289             if terminal:
 290                 self.status.world_target.add(pn)
 291
 292             # drop reference count now
 293             self.status.possible_world = None
 294             self.status.all_depends    = None
 295
 296     def myProgressCallback( self, x, y, f, from_cache ):
 297         """Update any tty with the progress change"""
 298         if os.isatty(sys.stdout.fileno()):
 299             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
 300             sys.stdout.flush()
 301         else:
 302             if x == 1:
 303                 sys.stdout.write("Parsing .bb files, please wait...")
 304                 sys.stdout.flush()
 305             if x == y:
 306                 sys.stdout.write("done.")
 307                 sys.stdout.flush()
 308
 309     def interactiveMode( self ):
 310         """Drop off into a shell"""
 311         try:
 312             from bb import shell
 313         except ImportError, details:
 314             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
 315         else:
 316             bb.data.update_data( self.configuration.data )
 317             bb.data.expandKeys( self.configuration.data )
 318             shell.start( self )
 319             sys.exit( 0 )
 320
 321     def parseConfigurationFile( self, afile ):
 322         try:
 323             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
 324
 325             # Add the handlers we inherited by INHERIT
 326             # we need to do this manually as it is not guranteed
 327             # we will pick up these classes... as we only INHERIT
 328             # on .inc and .bb files but not on .conf
 329             data = bb.data.createCopy( self.configuration.data )
 330             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
 331             for inherit in inherits:
 332                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
 333
 334             # FIXME: This assumes that we included at least one .inc file
 335             for var in bb.data.keys(data):
 336                 if bb.data.getVarFlag(var, 'handler', data):
 337                     bb.event.register(var,bb.data.getVar(var, data))
 338
 339             bb.fetch.fetcher_init(self.configuration.data)
 340
 341         except IOError:
 342             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
 343         except bb.parse.ParseError, details:
 344             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
 345
 346     def handleCollections( self, collections ):
 347         """Handle collections"""
 348         if collections:
 349             collection_list = collections.split()
 350             for c in collection_list:
 351                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
 352                 if regex == None:
 353                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
 354                     continue
 355                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
 356                 if priority == None:
 357                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
 358                     continue
 359                 try:
 360                     cre = re.compile(regex)
 361                 except re.error:
 362                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
 363                     continue
 364                 try:
 365                     pri = int(priority)
 366                     self.status.bbfile_config_priorities.append((cre, pri))
 367                 except ValueError:
 368                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
 369
 370     def buildSetVars(self):
 371         """
 372         Setup any variables needed before starting a build
 373         """
 374         if not bb.data.getVar("BUILDNAME", self.configuration.data):
 375             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
 376         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
 377
 378     def buildFile(self, buildfile):
 379         """
 380         Build the file matching regexp buildfile
 381         """
 382
 383         bf = os.path.abspath(buildfile)
 384         try:
 385             os.stat(bf)
 386         except OSError:
 387             (filelist, masked) = self.collect_bbfiles()
 388             regexp = re.compile(buildfile)
 389             matches = []
 390             for f in filelist:
 391                 if regexp.search(f) and os.path.isfile(f):
 392                     bf = f
 393                     matches.append(f)
 394             if len(matches) != 1:
 395                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
 396                 for f in matches:
 397                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
 398                 sys.exit(1)
 399             bf = matches[0]
 400
 401         bbfile_data = bb.parse.handle(bf, self.configuration.data)
 402
 403         # Remove stamp for target if force mode active
 404         if self.configuration.force:
 405             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
 406             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
 407
 408         item = bb.data.getVar('PN', bbfile_data, 1)
 409         try:
 410             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
 411         except bb.build.EventException:
 412             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
 413
 414         sys.exit(0)
 415
 416     def buildTargets(self, targets):
 417         """
 418         Attempt to build the targets specified
 419         """
 420
 421         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
 422         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
 423
 424         localdata = data.createCopy(self.configuration.data)
 425         bb.data.update_data(localdata)
 426         bb.data.expandKeys(localdata)
 427
 428         taskdata = bb.taskdata.TaskData(self.configuration.abort)
 429
 430         runlist = []
 431         try:
 432             for k in targets:
 433                 taskdata.add_provider(localdata, self.status, k)
 434                 runlist.append([k, "do_%s" % self.configuration.cmd])
 435             taskdata.add_unresolved(localdata, self.status)
 436         except bb.providers.NoProvider:
 437             sys.exit(1)
 438
 439         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 440         rq.prepare_runqueue()
 441         try:
 442             failures = rq.execute_runqueue()
 443         except runqueue.TaskFailure, fnids:
 444             for fnid in fnids:
 445                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
 446             sys.exit(1)
 447         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
 448
 449         sys.exit(0)
 450
 451     def updateCache(self):
 452         # Import Psyco if available and not disabled
 453         if not self.configuration.disable_psyco:
 454             try:
 455                 import psyco
 456             except ImportError:
 457                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
 458             else:
 459                 psyco.bind( self.parse_bbfiles )
 460         else:
 461             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
 462
 463         self.status = bb.cache.CacheData()
 464
 465         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
 466         self.status.ignored_dependencies = Set( ignore.split() )
 467
 468         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
 469
 470         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
 471         (filelist, masked) = self.collect_bbfiles()
 472         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
 473         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
 474
 475         self.buildDepgraph()
 476
 477     def cook(self):
 478         """
 479         We are building stuff here. We do the building
 480         from here. By default we try to execute task
 481         build.
 482         """
 483
 484         if self.configuration.show_environment:
 485             self.showEnvironment()
 486             sys.exit( 0 )
 487
 488         self.buildSetVars()
 489
 490         if self.configuration.interactive:
 491             self.interactiveMode()
 492
 493         if self.configuration.buildfile is not None:
 494             return self.buildFile(self.configuration.buildfile)
 495
 496         # initialise the parsing status now we know we will need deps
 497         self.updateCache()
 498
 499         if self.configuration.parse_only:
 500             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
 501             return 0
 502
 503         pkgs_to_build = self.configuration.pkgs_to_build
 504
 505         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
 506         if bbpkgs:
 507             pkgs_to_build.extend(bbpkgs.split())
 508         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
 509                              and not self.configuration.show_environment:
 510                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
 511                 print "for usage information."
 512                 sys.exit(0)
 513
 514         try:
 515             if self.configuration.show_versions:
 516                 self.showVersions()
 517                 sys.exit( 0 )
 518             if 'world' in pkgs_to_build:
 519                 self.buildWorldTargetList()
 520                 pkgs_to_build.remove('world')
 521                 for t in self.status.world_target:
 522                     pkgs_to_build.append(t)
 523
 524             if self.configuration.dot_graph:
 525                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
 526                 sys.exit( 0 )
 527
 528             return self.buildTargets(pkgs_to_build)
 529
 530         except KeyboardInterrupt:
 531             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
 532             sys.exit(1)
 533
 534     def get_bbfiles( self, path = os.getcwd() ):
 535         """Get list of default .bb files by reading out the current directory"""
 536         contents = os.listdir(path)
 537         bbfiles = []
 538         for f in contents:
 539             (root, ext) = os.path.splitext(f)
 540             if ext == ".bb":
 541                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
 542         return bbfiles
 543
 544     def find_bbfiles( self, path ):
 545         """Find all the .bb files in a directory"""
 546         from os.path import join
 547
 548         found = []
 549         for dir, dirs, files in os.walk(path):
 550             for ignored in ('SCCS', 'CVS', '.svn'):
 551                 if ignored in dirs:
 552                     dirs.remove(ignored)
 553             found += [join(dir,f) for f in files if f.endswith('.bb')]
 554
 555         return found
 556
 557     def collect_bbfiles( self ):
 558         """Collect all available .bb build files"""
 559         parsed, cached, skipped, masked = 0, 0, 0, 0
 560         self.bb_cache = bb.cache.init(self)
 561
 562         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
 563         data.setVar("BBFILES", " ".join(files), self.configuration.data)
 564
 565         if not len(files):
 566             files = self.get_bbfiles()
 567
 568         if not len(files):
 569             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
 570
 571         newfiles = []
 572         for f in files:
 573             if os.path.isdir(f):
 574                 dirfiles = self.find_bbfiles(f)
 575                 if dirfiles:
 576                     newfiles += dirfiles
 577                     continue
 578             newfiles += glob.glob(f) or [ f ]
 579
 580         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
 581
 582         if not bbmask:
 583             return (newfiles, 0)
 584
 585         try:
 586             bbmask_compiled = re.compile(bbmask)
 587         except sre_constants.error:
 588             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
 589
 590         finalfiles = []
 591         for i in xrange( len( newfiles ) ):
 592             f = newfiles[i]
 593             if bbmask and bbmask_compiled.search(f):
 594                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
 595                 masked += 1
 596                 continue
 597             finalfiles.append(f)
 598
 599         return (finalfiles, masked)
 600
 601     def parse_bbfiles(self, filelist, masked, progressCallback = None):
 602         parsed, cached, skipped, error = 0, 0, 0, 0
 603         for i in xrange( len( filelist ) ):
 604             f = filelist[i]
 605
 606             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
 607
 608             # read a file's metadata
 609             try:
 610                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
 611                 if skip:
 612                     skipped += 1
 613                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
 614                     self.bb_cache.skip(f)
 615                     continue
 616                 elif fromCache: cached += 1
 617                 else: parsed += 1
 618                 deps = None
 619
 620                 # Disabled by RP as was no longer functional
 621                 # allow metadata files to add items to BBFILES
 622                 #data.update_data(self.pkgdata[f])
 623                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
 624                 #if addbbfiles:
 625                 #    for aof in addbbfiles.split():
 626                 #        if not files.count(aof):
 627                 #            if not os.path.isabs(aof):
 628                 #                aof = os.path.join(os.path.dirname(f),aof)
 629                 #            files.append(aof)
 630
 631                 self.bb_cache.handle_data(f, self.status)
 632
 633                 # now inform the caller
 634                 if progressCallback is not None:
 635                     progressCallback( i + 1, len( filelist ), f, fromCache )
 636
 637             except IOError, e:
 638                 self.bb_cache.remove(f)
 639                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
 640                 pass
 641             except KeyboardInterrupt:
 642                 self.bb_cache.sync()
 643                 raise
 644             except Exception, e:
 645                 error += 1
 646                 self.bb_cache.remove(f)
 647                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
 648             except:
 649                 self.bb_cache.remove(f)
 650                 raise
 651
 652         if progressCallback is not None:
 653             print "\r" # need newline after Handling Bitbake files message
 654             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
 655
 656         self.bb_cache.sync()
 657
 658         if error > 0:
 659             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")