lib/bb/cooker.py

   1 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 #
   5 # Copyright (C) 2003, 2004  Chris Larson
   6 # Copyright (C) 2003, 2004  Phil Blundell
   7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
   8 # Copyright (C) 2005        Holger Hans Peter Freyther
   9 # Copyright (C) 2005        ROAD GmbH
  10 # Copyright (C) 2006 - 2007 Richard Purdie
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License version 2 as
  14 # published by the Free Software Foundation.
  15 #
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20 #
  21 # You should have received a copy of the GNU General Public License along
  22 # with this program; if not, write to the Free Software Foundation, Inc.,
  23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  24
  25 import sys, os, getopt, glob, copy, os.path, re, time
  26 import bb
  27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
  28 from bb import command
  29 import itertools, sre_constants
  30
  31 class MultipleMatches(Exception):
  32     """
  33     Exception raised when multiple file matches are found
  34     """
  35
  36 class ParsingErrorsFound(Exception):
  37     """
  38     Exception raised when parsing errors are found
  39     """
  40
  41 class NothingToBuild(Exception):
  42     """
  43     Exception raised when there is nothing to build
  44     """
  45
  46
  47 # Different states cooker can be in
  48 cookerClean = 1
  49 cookerParsing = 2
  50 cookerParsed = 3
  51
  52 # Different action states the cooker can be in
  53 cookerRun = 1           # Cooker is running normally
  54 cookerShutdown = 2      # Active tasks should be brought to a controlled stop
  55 cookerStop = 3          # Stop, now!
  56
  57 #============================================================================#
  58 # BBCooker
  59 #============================================================================#
  60 class BBCooker:
  61     """
  62     Manages one bitbake build run
  63     """
  64
  65     def __init__(self, configuration, server):
  66         self.status = None
  67
  68         self.cache = None
  69         self.bb_cache = None
  70
  71         self.server = server.BitBakeServer(self)
  72
  73         self.configuration = configuration
  74
  75         if self.configuration.verbose:
  76             bb.msg.set_verbose(True)
  77
  78         if self.configuration.debug:
  79             bb.msg.set_debug_level(self.configuration.debug)
  80         else:
  81             bb.msg.set_debug_level(0)
  82
  83         if self.configuration.debug_domains:
  84             bb.msg.set_debug_domains(self.configuration.debug_domains)
  85
  86         self.configuration.data = bb.data.init()
  87
  88         bb.data.inheritFromOS(self.configuration.data)
  89
  90         self.parseConfigurationFiles(self.configuration.file)
  91
  92         if not self.configuration.cmd:
  93             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
  94
  95         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
  96         if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
  97             self.configuration.pkgs_to_build.extend(bbpkgs.split())
  98
  99         #
 100         # Special updated configuration we use for firing events
 101         #
 102         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
 103         bb.data.update_data(self.configuration.event_data)
 104
 105         # TOSTOP must not be set or our children will hang when they output
 106         fd = sys.stdout.fileno()
 107         if os.isatty(fd):
 108             import termios
 109             tcattr = termios.tcgetattr(fd)
 110             if tcattr[3] & termios.TOSTOP:
 111                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
 112                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
 113                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
 114
 115         self.command = bb.command.Command(self)
 116         self.cookerState = cookerClean
 117         self.cookerAction = cookerRun
 118
 119     def parseConfiguration(self):
 120
 121
 122         # Change nice level if we're asked to
 123         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
 124         if nice:
 125             curnice = os.nice(0)
 126             nice = int(nice) - curnice
 127             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
 128
 129     def parseCommandLine(self):
 130         # Parse any commandline into actions
 131         if self.configuration.show_environment:
 132             self.commandlineAction = None
 133
 134             if 'world' in self.configuration.pkgs_to_build:
 135                 bb.error("'world' is not a valid target for --environment.")
 136             elif len(self.configuration.pkgs_to_build) > 1:
 137                 bb.error("Only one target can be used with the --environment option.")
 138             elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
 139                 bb.error("No target should be used with the --environment and --buildfile options.")
 140             elif len(self.configuration.pkgs_to_build) > 0:
 141                 self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
 142             else:
 143                 self.commandlineAction = ["showEnvironment", self.configuration.buildfile]
 144         elif self.configuration.buildfile is not None:
 145             self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
 146         elif self.configuration.revisions_changed:
 147             self.commandlineAction = ["compareRevisions"]
 148         elif self.configuration.show_versions:
 149             self.commandlineAction = ["showVersions"]
 150         elif self.configuration.parse_only:
 151             self.commandlineAction = ["parseFiles"]
 152         # FIXME - implement
 153         #elif self.configuration.interactive:
 154         #    self.interactiveMode()
 155         elif self.configuration.dot_graph:
 156             if self.configuration.pkgs_to_build:
 157                 self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
 158             else:
 159                 self.commandlineAction = None
 160                 bb.error("Please specify a package name for dependency graph generation.")
 161         else:
 162             if self.configuration.pkgs_to_build:
 163                 self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
 164             else:
 165                 self.commandlineAction = None
 166                 bb.error("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
 167
 168     def runCommands(self, server, data, abort):
 169         """
 170         Run any queued asynchronous command
 171         This is done by the idle handler so it runs in true context rather than
 172         tied to any UI.
 173         """
 174
 175         return self.command.runAsyncCommand()
 176
 177     def tryBuildPackage(self, fn, item, task, the_data):
 178         """
 179         Build one task of a package, optionally build following task depends
 180         """
 181         try:
 182             if not self.configuration.dry_run:
 183                 bb.build.exec_task('do_%s' % task, the_data)
 184             return True
 185         except bb.build.FuncFailed:
 186             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
 187             raise
 188         except bb.build.EventException, e:
 189             event = e.args[1]
 190             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
 191             raise
 192
 193     def tryBuild(self, fn, task):
 194         """
 195         Build a provider and its dependencies.
 196         build_depends is a list of previous build dependencies (not runtime)
 197         If build_depends is empty, we're dealing with a runtime depends
 198         """
 199
 200         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
 201
 202         item = self.status.pkg_fn[fn]
 203
 204         #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
 205         #    return True
 206
 207         return self.tryBuildPackage(fn, item, task, the_data)
 208
 209     def showVersions(self):
 210
 211         # Need files parsed
 212         self.updateCache()
 213
 214         pkg_pn = self.status.pkg_pn
 215         preferred_versions = {}
 216         latest_versions = {}
 217
 218         # Sort by priority
 219         for pn in pkg_pn:
 220             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
 221             preferred_versions[pn] = (pref_ver, pref_file)
 222             latest_versions[pn] = (last_ver, last_file)
 223
 224         bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version"))
 225         bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "================="))
 226
 227         for p in sorted(pkg_pn):
 228             pref = preferred_versions[p]
 229             latest = latest_versions[p]
 230
 231             prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
 232             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
 233
 234             if pref == latest:
 235                 prefstr = ""
 236
 237             bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr))
 238
 239     def compareRevisions(self):
 240         ret = bb.fetch.fetcher_compare_revisons(self.configuration.data)
 241         bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data)
 242
 243     def showEnvironment(self, buildfile = None, pkgs_to_build = []):
 244         """
 245         Show the outer or per-package environment
 246         """
 247         fn = None
 248         envdata = None
 249
 250         if buildfile:
 251             self.cb = None
 252             self.bb_cache = bb.cache.init(self)
 253             fn = self.matchFile(buildfile)
 254         elif len(pkgs_to_build) == 1:
 255             self.updateCache()
 256
 257             localdata = data.createCopy(self.configuration.data)
 258             bb.data.update_data(localdata)
 259             bb.data.expandKeys(localdata)
 260
 261             taskdata = bb.taskdata.TaskData(self.configuration.abort)
 262             taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
 263             taskdata.add_unresolved(localdata, self.status)
 264
 265             targetid = taskdata.getbuild_id(pkgs_to_build[0])
 266             fnid = taskdata.build_targets[targetid][0]
 267             fn = taskdata.fn_index[fnid]
 268         else:
 269             envdata = self.configuration.data
 270
 271         if fn:
 272             try:
 273                 envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
 274             except IOError, e:
 275                 bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
 276                 raise
 277             except Exception, e:
 278                 bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
 279                 raise
 280
 281         class dummywrite:
 282             def __init__(self):
 283                 self.writebuf = ""
 284             def write(self, output):
 285                 self.writebuf = self.writebuf + output
 286
 287         # emit variables and shell functions
 288         try:
 289             data.update_data(envdata)
 290             wb = dummywrite()
 291             data.emit_env(wb, envdata, True)
 292             bb.msg.plain(wb.writebuf)
 293         except Exception, e:
 294             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
 295         # emit the metadata which isnt valid shell
 296         data.expandKeys(envdata)
 297         for e in envdata.keys():
 298             if data.getVarFlag( e, 'python', envdata ):
 299                 bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
 300
 301     def generateDepTreeData(self, pkgs_to_build, task):
 302         """
 303         Create a dependency tree of pkgs_to_build, returning the data.
 304         """
 305
 306         # Need files parsed
 307         self.updateCache()
 308
 309         # If we are told to do the None task then query the default task
 310         if (task == None):
 311             task = self.configuration.cmd
 312
 313         pkgs_to_build = self.checkPackages(pkgs_to_build)
 314
 315         localdata = data.createCopy(self.configuration.data)
 316         bb.data.update_data(localdata)
 317         bb.data.expandKeys(localdata)
 318         taskdata = bb.taskdata.TaskData(self.configuration.abort)
 319
 320         runlist = []
 321         for k in pkgs_to_build:
 322             taskdata.add_provider(localdata, self.status, k)
 323             runlist.append([k, "do_%s" % task])
 324         taskdata.add_unresolved(localdata, self.status)
 325
 326         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 327         rq.prepare_runqueue()
 328
 329         seen_fnids = []
 330         depend_tree = {}
 331         depend_tree["depends"] = {}
 332         depend_tree["tdepends"] = {}
 333         depend_tree["pn"] = {}
 334         depend_tree["rdepends-pn"] = {}
 335         depend_tree["packages"] = {}
 336         depend_tree["rdepends-pkg"] = {}
 337         depend_tree["rrecs-pkg"] = {}
 338
 339         for task in range(len(rq.runq_fnid)):
 340             taskname = rq.runq_task[task]
 341             fnid = rq.runq_fnid[task]
 342             fn = taskdata.fn_index[fnid]
 343             pn = self.status.pkg_fn[fn]
 344             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
 345             if pn not in depend_tree["pn"]:
 346                 depend_tree["pn"][pn] = {}
 347                 depend_tree["pn"][pn]["filename"] = fn
 348                 depend_tree["pn"][pn]["version"] = version
 349             for dep in rq.runq_depends[task]:
 350                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
 351                 deppn = self.status.pkg_fn[depfn]
 352                 dotname = "%s.%s" % (pn, rq.runq_task[task])
 353                 if not dotname in depend_tree["tdepends"]:
 354                     depend_tree["tdepends"][dotname] = []
 355                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep]))
 356             if fnid not in seen_fnids:
 357                 seen_fnids.append(fnid)
 358                 packages = []
 359
 360                 depend_tree["depends"][pn] = []
 361                 for dep in taskdata.depids[fnid]:
 362                     depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
 363
 364                 depend_tree["rdepends-pn"][pn] = []
 365                 for rdep in taskdata.rdepids[fnid]:
 366                         depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
 367
 368                 rdepends = self.status.rundeps[fn]
 369                 for package in rdepends:
 370                     depend_tree["rdepends-pkg"][package] = []
 371                     for rdepend in bb.utils.explode_deps(rdepends[package]):
 372                         depend_tree["rdepends-pkg"][package].append(rdepend)
 373                     packages.append(package)
 374
 375                 rrecs = self.status.runrecs[fn]
 376                 for package in rrecs:
 377                     depend_tree["rrecs-pkg"][package] = []
 378                     for rdepend in bb.utils.explode_deps(rrecs[package]):
 379                         depend_tree["rrecs-pkg"][package].append(rdepend)
 380                     if not package in packages:
 381                         packages.append(package)
 382
 383                 for package in packages:
 384                     if package not in depend_tree["packages"]:
 385                         depend_tree["packages"][package] = {}
 386                         depend_tree["packages"][package]["pn"] = pn
 387                         depend_tree["packages"][package]["filename"] = fn
 388                         depend_tree["packages"][package]["version"] = version
 389
 390         return depend_tree
 391
 392
 393     def generateDepTreeEvent(self, pkgs_to_build, task):
 394         """
 395         Create a task dependency graph of pkgs_to_build.
 396         Generate an event with the result
 397         """
 398         depgraph = self.generateDepTreeData(pkgs_to_build, task)
 399         bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
 400
 401     def generateDotGraphFiles(self, pkgs_to_build, task):
 402         """
 403         Create a task dependency graph of pkgs_to_build.
 404         Save the result to a set of .dot files.
 405         """
 406
 407         depgraph = self.generateDepTreeData(pkgs_to_build, task)
 408
 409         # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
 410         depends_file = file('pn-depends.dot', 'w' )
 411         print >> depends_file, "digraph depends {"
 412         for pn in depgraph["pn"]:
 413             fn = depgraph["pn"][pn]["filename"]
 414             version = depgraph["pn"][pn]["version"]
 415             print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
 416         for pn in depgraph["depends"]:
 417             for depend in depgraph["depends"][pn]:
 418                 print >> depends_file, '"%s" -> "%s"' % (pn, depend)
 419         for pn in depgraph["rdepends-pn"]:
 420             for rdepend in depgraph["rdepends-pn"][pn]:
 421                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend)
 422         print >> depends_file,  "}"
 423         bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
 424
 425         depends_file = file('package-depends.dot', 'w' )
 426         print >> depends_file, "digraph depends {"
 427         for package in depgraph["packages"]:
 428             pn = depgraph["packages"][package]["pn"]
 429             fn = depgraph["packages"][package]["filename"]
 430             version = depgraph["packages"][package]["version"]
 431             if package == pn:
 432                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)
 433             else:
 434                 print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
 435             for depend in depgraph["depends"][pn]:
 436                 print >> depends_file, '"%s" -> "%s"' % (package, depend)
 437         for package in depgraph["rdepends-pkg"]:
 438             for rdepend in depgraph["rdepends-pkg"][package]:
 439                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
 440         for package in depgraph["rrecs-pkg"]:
 441             for rdepend in depgraph["rrecs-pkg"][package]:
 442                 print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
 443         print >> depends_file,  "}"
 444         bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
 445
 446         tdepends_file = file('task-depends.dot', 'w' )
 447         print >> tdepends_file, "digraph depends {"
 448         for task in depgraph["tdepends"]:
 449             (pn, taskname) = task.rsplit(".", 1)
 450             fn = depgraph["pn"][pn]["filename"]
 451             version = depgraph["pn"][pn]["version"]
 452             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
 453             for dep in depgraph["tdepends"][task]:
 454                 print >> tdepends_file, '"%s" -> "%s"' % (task, dep)
 455         print >> tdepends_file,  "}"
 456         bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
 457
 458     def buildDepgraph( self ):
 459         all_depends = self.status.all_depends
 460         pn_provides = self.status.pn_provides
 461
 462         localdata = data.createCopy(self.configuration.data)
 463         bb.data.update_data(localdata)
 464         bb.data.expandKeys(localdata)
 465
 466         def calc_bbfile_priority(filename):
 467             for (regex, pri) in self.status.bbfile_config_priorities:
 468                 if regex.match(filename):
 469                     return pri
 470             return 0
 471
 472         # Handle PREFERRED_PROVIDERS
 473         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
 474             try:
 475                 (providee, provider) = p.split(':')
 476             except:
 477                 bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
 478                 continue
 479             if providee in self.status.preferred and self.status.preferred[providee] != provider:
 480                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
 481             self.status.preferred[providee] = provider
 482
 483         # Calculate priorities for each file
 484         for p in self.status.pkg_fn:
 485             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
 486
 487     def buildWorldTargetList(self):
 488         """
 489          Build package list for "bitbake world"
 490         """
 491         all_depends = self.status.all_depends
 492         pn_provides = self.status.pn_provides
 493         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
 494         for f in self.status.possible_world:
 495             terminal = True
 496             pn = self.status.pkg_fn[f]
 497
 498             for p in pn_provides[pn]:
 499                 if p.startswith('virtual/'):
 500                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
 501                     terminal = False
 502                     break
 503                 for pf in self.status.providers[p]:
 504                     if self.status.pkg_fn[pf] != pn:
 505                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
 506                         terminal = False
 507                         break
 508             if terminal:
 509                 self.status.world_target.add(pn)
 510
 511             # drop reference count now
 512             self.status.possible_world = None
 513             self.status.all_depends    = None
 514
 515     def interactiveMode( self ):
 516         """Drop off into a shell"""
 517         try:
 518             from bb import shell
 519         except ImportError, details:
 520             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
 521         else:
 522             shell.start( self )
 523
 524     def _findLayerConf(self):
 525         path = os.getcwd()
 526         while path != "/":
 527             bblayers = os.path.join(path, "conf", "bblayers.conf")
 528             if os.path.exists(bblayers):
 529                 return bblayers
 530
 531             path, _ = os.path.split(path)
 532
 533     def parseConfigurationFiles(self, files):
 534         try:
 535             data = self.configuration.data
 536             for f in files:
 537                 data = bb.parse.handle(f, data)
 538
 539             layerconf = self._findLayerConf()
 540             if layerconf:
 541                 bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf)
 542                 data = bb.parse.handle(layerconf, data)
 543
 544                 layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
 545
 546                 data = bb.data.createCopy(data)
 547                 for layer in layers:
 548                     bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer)
 549                     bb.data.setVar('LAYERDIR', layer, data)
 550                     data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data)
 551
 552                     # XXX: Hack, relies on the local keys of the datasmart
 553                     # instance being stored in the 'dict' attribute and makes
 554                     # assumptions about how variable expansion works, but
 555                     # there's no better way to force an expansion of a single
 556                     # variable across the datastore today, and this at least
 557                     # lets us reference LAYERDIR without having to immediately
 558                     # eval all our variables that use it.
 559                     for key in data.dict:
 560                         if key != "_data":
 561                             value = data.getVar(key, False)
 562                             if value and "${LAYERDIR}" in value:
 563                                 data.setVar(key, value.replace("${LAYERDIR}", layer))
 564
 565                 bb.data.delVar('LAYERDIR', data)
 566
 567             if not data.getVar("BBPATH", True):
 568                 bb.fatal("The BBPATH variable is not set")
 569
 570             data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data)
 571
 572             self.configuration.data = data
 573
 574             # Handle any INHERITs and inherit the base class
 575             inherits  = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
 576             for inherit in inherits:
 577                 self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
 578
 579             # Nomally we only register event handlers at the end of parsing .bb files
 580             # We register any handlers we've found so far here...
 581             for var in data.getVar('__BBHANDLERS', self.configuration.data) or []:
 582                 bb.event.register(var,bb.data.getVar(var, self.configuration.data))
 583
 584             bb.fetch.fetcher_init(self.configuration.data)
 585
 586             bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
 587
 588
 589         except IOError, e:
 590             bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
 591         except bb.parse.ParseError, details:
 592             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) )
 593
 594     def handleCollections( self, collections ):
 595         """Handle collections"""
 596         if collections:
 597             collection_list = collections.split()
 598             for c in collection_list:
 599                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
 600                 if regex == None:
 601                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
 602                     continue
 603                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
 604                 if priority == None:
 605                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
 606                     continue
 607                 try:
 608                     cre = re.compile(regex)
 609                 except re.error:
 610                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
 611                     continue
 612                 try:
 613                     pri = int(priority)
 614                     self.status.bbfile_config_priorities.append((cre, pri))
 615                 except ValueError:
 616                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
 617
 618     def buildSetVars(self):
 619         """
 620         Setup any variables needed before starting a build
 621         """
 622         if not bb.data.getVar("BUILDNAME", self.configuration.data):
 623             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
 624         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data)
 625
 626     def matchFiles(self, buildfile):
 627         """
 628         Find the .bb files which match the expression in 'buildfile'.
 629         """
 630
 631         bf = os.path.abspath(buildfile)
 632         try:
 633             os.stat(bf)
 634             return [bf]
 635         except OSError:
 636             (filelist, masked) = self.collect_bbfiles()
 637             regexp = re.compile(buildfile)
 638             matches = []
 639             for f in filelist:
 640                 if regexp.search(f) and os.path.isfile(f):
 641                     bf = f
 642                     matches.append(f)
 643             return matches
 644
 645     def matchFile(self, buildfile):
 646         """
 647         Find the .bb file which matches the expression in 'buildfile'.
 648         Raise an error if multiple files
 649         """
 650         matches = self.matchFiles(buildfile)
 651         if len(matches) != 1:
 652             bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
 653             for f in matches:
 654                 bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
 655             raise MultipleMatches
 656         return matches[0]
 657
 658     def buildFile(self, buildfile, task):
 659         """
 660         Build the file matching regexp buildfile
 661         """
 662
 663         # Parse the configuration here. We need to do it explicitly here since
 664         # buildFile() doesn't use the cache
 665         self.parseConfiguration()
 666
 667         # If we are told to do the None task then query the default task
 668         if (task == None):
 669             task = self.configuration.cmd
 670
 671         self.bb_cache = bb.cache.init(self)
 672         self.status = bb.cache.CacheData()
 673
 674         (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile)
 675         buildfile = self.matchFile(fn)
 676         fn = self.bb_cache.realfn2virtual(buildfile, cls)
 677
 678         self.buildSetVars()
 679
 680         # Load data into the cache for fn and parse the loaded cache data
 681         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
 682         self.bb_cache.setData(fn, buildfile, the_data)
 683         self.bb_cache.handle_data(fn, self.status)
 684
 685         # Tweak some variables
 686         item = self.bb_cache.getVar('PN', fn, True)
 687         self.status.ignored_dependencies = set()
 688         self.status.bbfile_priority[fn] = 1
 689
 690         # Remove external dependencies
 691         self.status.task_deps[fn]['depends'] = {}
 692         self.status.deps[fn] = []
 693         self.status.rundeps[fn] = []
 694         self.status.runrecs[fn] = []
 695
 696         # Remove stamp for target if force mode active
 697         if self.configuration.force:
 698             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn))
 699             bb.build.del_stamp('do_%s' % task, self.status, fn)
 700
 701         # Setup taskdata structure
 702         taskdata = bb.taskdata.TaskData(self.configuration.abort)
 703         taskdata.add_provider(self.configuration.data, self.status, item)
 704
 705         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
 706         bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data)
 707
 708         # Execute the runqueue
 709         runlist = [[item, "do_%s" % task]]
 710
 711         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 712
 713         def buildFileIdle(server, rq, abort):
 714
 715             if abort or self.cookerAction == cookerStop:
 716                 rq.finish_runqueue(True)
 717             elif self.cookerAction == cookerShutdown:
 718                 rq.finish_runqueue(False)
 719             failures = 0
 720             try:
 721                 retval = rq.execute_runqueue()
 722             except runqueue.TaskFailure, fnids:
 723                 for fnid in fnids:
 724                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
 725                     failures = failures + 1
 726                 retval = False
 727             if not retval:
 728                 bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
 729                 self.command.finishAsyncCommand()
 730                 return False
 731             return 0.5
 732
 733         self.server.register_idle_function(buildFileIdle, rq)
 734
 735     def buildTargets(self, targets, task):
 736         """
 737         Attempt to build the targets specified
 738         """
 739
 740         # Need files parsed
 741         self.updateCache()
 742
 743         # If we are told to do the NULL task then query the default task
 744         if (task == None):
 745             task = self.configuration.cmd
 746
 747         targets = self.checkPackages(targets)
 748
 749         def buildTargetsIdle(server, rq, abort):
 750
 751             if abort or self.cookerAction == cookerStop:
 752                 rq.finish_runqueue(True)
 753             elif self.cookerAction == cookerShutdown:
 754                 rq.finish_runqueue(False)
 755             failures = 0
 756             try:
 757                 retval = rq.execute_runqueue()
 758             except runqueue.TaskFailure, fnids:
 759                 for fnid in fnids:
 760                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
 761                     failures = failures + 1
 762                 retval = False
 763             if not retval:
 764                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data)
 765                 self.command.finishAsyncCommand()
 766                 return None
 767             return 0.5
 768
 769         self.buildSetVars()
 770
 771         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
 772         bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data)
 773
 774         localdata = data.createCopy(self.configuration.data)
 775         bb.data.update_data(localdata)
 776         bb.data.expandKeys(localdata)
 777
 778         taskdata = bb.taskdata.TaskData(self.configuration.abort)
 779
 780         runlist = []
 781         for k in targets:
 782             taskdata.add_provider(localdata, self.status, k)
 783             runlist.append([k, "do_%s" % task])
 784         taskdata.add_unresolved(localdata, self.status)
 785
 786         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 787
 788         self.server.register_idle_function(buildTargetsIdle, rq)
 789
 790     def updateCache(self):
 791
 792         if self.cookerState == cookerParsed:
 793             return
 794
 795         if self.cookerState != cookerParsing:
 796
 797             self.parseConfiguration ()
 798
 799             # Import Psyco if available and not disabled
 800             import platform
 801             if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
 802                 if not self.configuration.disable_psyco:
 803                     try:
 804                         import psyco
 805                     except ImportError:
 806                         bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
 807                     else:
 808                         psyco.bind( CookerParser.parse_next )
 809                 else:
 810                     bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
 811
 812             self.status = bb.cache.CacheData()
 813
 814             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
 815             self.status.ignored_dependencies = set(ignore.split())
 816
 817             for dep in self.configuration.extra_assume_provided:
 818                 self.status.ignored_dependencies.add(dep)
 819
 820             self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
 821
 822             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
 823             (filelist, masked) = self.collect_bbfiles()
 824             bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
 825
 826             self.parser = CookerParser(self, filelist, masked)
 827             self.cookerState = cookerParsing
 828
 829         if not self.parser.parse_next():
 830             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
 831             self.buildDepgraph()
 832             self.cookerState = cookerParsed
 833             return None
 834
 835         return True
 836
 837     def checkPackages(self, pkgs_to_build):
 838
 839         if len(pkgs_to_build) == 0:
 840             raise NothingToBuild
 841
 842         if 'world' in pkgs_to_build:
 843             self.buildWorldTargetList()
 844             pkgs_to_build.remove('world')
 845             for t in self.status.world_target:
 846                 pkgs_to_build.append(t)
 847
 848         return pkgs_to_build
 849
 850     def get_bbfiles( self, path = os.getcwd() ):
 851         """Get list of default .bb files by reading out the current directory"""
 852         contents = os.listdir(path)
 853         bbfiles = []
 854         for f in contents:
 855             (root, ext) = os.path.splitext(f)
 856             if ext == ".bb":
 857                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
 858         return bbfiles
 859
 860     def find_bbfiles( self, path ):
 861         """Find all the .bb files in a directory"""
 862         from os.path import join
 863
 864         found = []
 865         for dir, dirs, files in os.walk(path):
 866             for ignored in ('SCCS', 'CVS', '.svn'):
 867                 if ignored in dirs:
 868                     dirs.remove(ignored)
 869             found += [join(dir,f) for f in files if f.endswith('.bb')]
 870
 871         return found
 872
 873     def collect_bbfiles( self ):
 874         """Collect all available .bb build files"""
 875         parsed, cached, skipped, masked = 0, 0, 0, 0
 876         self.bb_cache = bb.cache.init(self)
 877
 878         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
 879         data.setVar("BBFILES", " ".join(files), self.configuration.data)
 880
 881         if not len(files):
 882             files = self.get_bbfiles()
 883
 884         if not len(files):
 885             bb.msg.error(bb.msg.domain.Collection, "no recipe files to build, check your BBPATH and BBFILES?")
 886             bb.event.fire(CookerExit(), self.configuration.event_data)
 887
 888         newfiles = set()
 889         for f in files:
 890             if os.path.isdir(f):
 891                 dirfiles = self.find_bbfiles(f)
 892                 if dirfiles:
 893                     newfiles.update(dirfiles)
 894                     continue
 895             else:
 896                 globbed = glob.glob(f)
 897                 if not globbed and os.path.exists(f):
 898                     globbed = [f]
 899                 newfiles.update(globbed)
 900
 901         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
 902
 903         if not bbmask:
 904             return (list(newfiles), 0)
 905
 906         try:
 907             bbmask_compiled = re.compile(bbmask)
 908         except sre_constants.error:
 909             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
 910
 911         finalfiles = []
 912         for f in newfiles:
 913             if bbmask_compiled.search(f):
 914                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
 915                 masked += 1
 916                 continue
 917             finalfiles.append(f)
 918
 919         return (finalfiles, masked)
 920
 921     def serve(self):
 922
 923         # Empty the environment. The environment will be populated as
 924         # necessary from the data store.
 925         bb.utils.empty_environment()
 926
 927         if self.configuration.profile:
 928             try:
 929                 import cProfile as profile
 930             except:
 931                 import profile
 932
 933             profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
 934
 935             # Redirect stdout to capture profile information
 936             pout = open('profile.log.processed', 'w')
 937             so = sys.stdout.fileno()
 938             os.dup2(pout.fileno(), so)
 939
 940             import pstats
 941             p = pstats.Stats('profile.log')
 942             p.sort_stats('time')
 943             p.print_stats()
 944             p.print_callers()
 945             p.sort_stats('cumulative')
 946             p.print_stats()
 947
 948             os.dup2(so, pout.fileno())
 949             pout.flush()
 950             pout.close()
 951         else:
 952             self.server.serve_forever()
 953
 954         bb.event.fire(CookerExit(), self.configuration.event_data)
 955
 956 class CookerExit(bb.event.Event):
 957     """
 958     Notify clients of the Cooker shutdown
 959     """
 960
 961     def __init__(self):
 962         bb.event.Event.__init__(self)
 963
 964 class CookerParser:
 965     def __init__(self, cooker, filelist, masked):
 966         # Internal data
 967         self.filelist = filelist
 968         self.cooker = cooker
 969
 970         # Accounting statistics
 971         self.parsed = 0
 972         self.cached = 0
 973         self.error = 0
 974         self.masked = masked
 975         self.total = len(filelist)
 976
 977         self.skipped = 0
 978         self.virtuals = 0
 979
 980         # Pointer to the next file to parse
 981         self.pointer = 0
 982
 983     def parse_next(self):
 984         if self.pointer < len(self.filelist):
 985             f = self.filelist[self.pointer]
 986             cooker = self.cooker
 987
 988             try:
 989                 fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
 990                 if fromCache:
 991                     self.cached += 1
 992                 else:
 993                     self.parsed += 1
 994
 995                 self.skipped += skipped
 996                 self.virtuals += virtuals
 997
 998             except IOError, e:
 999                 self.error += 1
1000                 cooker.bb_cache.remove(f)
1001                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
1002                 pass
1003             except KeyboardInterrupt:
1004                 cooker.bb_cache.remove(f)
1005                 cooker.bb_cache.sync()
1006                 raise
1007             except Exception, e:
1008                 self.error += 1
1009                 cooker.bb_cache.remove(f)
1010                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
1011             except:
1012                 cooker.bb_cache.remove(f)
1013                 raise
1014             finally:
1015                 bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data)
1016
1017             self.pointer += 1
1018
1019         if self.pointer >= self.total:
1020             cooker.bb_cache.sync()
1021             if self.error > 0:
1022                 raise ParsingErrorsFound
1023             return False
1024         return True
1025