2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
39 __depcmds = { "clean": None,
44 bbfile_config_priorities = []
48 def handle_options( args ):
49 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
50 usage = """%prog [options] [package ...]
52 Executes the specified task (default is 'build') for a given set of BitBake files.
53 It expects that BBFILES is defined, which is a space seperated list of files to
54 be executed. BBFILES does support wildcards.
55 Default packages to be executed are all packages in BBFILES.
56 Default BBFILES are the .bb files in the current directory.""" )
58 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
59 action = "store_false", dest = "abort", default = True )
61 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
62 action = "store_true", dest = "force", default = False )
65 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
66 action = "store", dest = "cmd", default = "build" )
68 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
69 action = "append", dest = "file", default = [] )
71 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
72 action = "store_true", dest = "verbose", default = False )
74 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
75 action = "store_true", dest = "dry_run", default = False )
77 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
78 action = "store_true", dest = "parse_only", default = False )
80 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
81 action = "store_true", dest = "disable_psyco", default = False )
83 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
84 action = "store_true", dest = "show_versions", default = False )
86 options, args = parser.parse_args( args )
87 return options, args[1:]
89 def try_build(fn, virtual):
90 if fn in __building_list:
91 bb.error("%s depends on itself (eventually)" % fn)
92 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
95 __building_list.append(fn)
97 the_data = make.pkgdata[fn]
98 item = bb.data.getVar('PN', the_data, 1)
99 pathstr = "%s (%s)" % (item, virtual)
100 __build_path.append(pathstr)
102 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
103 if make.options.verbose:
104 bb.note("current path: %s" % (" -> ".join(__build_path)))
105 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
111 oldcmd = make.options.cmd
112 make.options.cmd = __depcmd
114 for d in depends_list:
115 if d in __ignored_dependencies:
119 if buildPackage(d) == 0:
120 bb.error("dependency %s (for %s) not satisfied" % (d,item))
122 if make.options.abort:
126 make.options.cmd = oldcmd
132 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
134 __stats["attempt"] += 1
135 if not make.options.dry_run:
136 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
137 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
138 __build_cache.append(fn)
140 except bb.build.FuncFailed:
142 bb.error("task stack execution failed")
143 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
144 __build_cache_fail.append(fn)
146 except bb.build.EventException:
148 (type, value, traceback) = sys.exc_info()
150 bb.error("%s event exception, aborting" % bb.event.getName(e))
151 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
152 __build_cache_fail.append(fn)
155 __building_list.remove(fn)
156 __build_path.remove(pathstr)
160 preferred_versions = {}
163 for p in make.pkgdata.keys():
164 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
165 if not pkg_pn.has_key(pn):
170 for pn in pkg_pn.keys():
174 priority = bbfile_priority[f]
175 if not priorities.has_key(priority):
176 priorities[priority] = []
177 priorities[priority].append(f)
178 p_list = priorities.keys()
179 p_list.sort(lambda a, b: a - b)
182 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
184 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
185 # version. If not, find the latest version provided by an bbfile in the
186 # highest-priority set.
187 for pn in pkg_pn.keys():
188 preferred_file = None
190 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
193 m = re.match('(.*)_(.*)', preferred_v)
195 preferred_v = m.group(1)
196 preferred_r = m.group(2)
198 for file_set in pkg_pn[pn]:
200 the_data = make.pkgdata[f]
201 pv = bb.data.getVar('PV', the_data, 1)
202 pr = bb.data.getVar('PR', the_data, 1)
203 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
205 preferred_ver = (pv, pr)
210 pv_str = '%s-%s' % (preferred_v, preferred_r)
213 if preferred_file is None:
214 bb.note("preferred version %s of %s not available" % (pv_str, pn))
216 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
218 # get highest priority file set
219 files = pkg_pn[pn][0]
224 the_data = make.pkgdata[f]
225 pv = bb.data.getVar('PV', the_data, 1)
226 pr = bb.data.getVar('PR', the_data, 1)
227 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
229 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
233 if preferred_file is None:
234 preferred_file = latest_f
235 preferred_ver = latest
237 preferred_versions[pn] = (preferred_ver, preferred_file)
238 latest_versions[pn] = (latest, latest_f)
240 pkg_list = pkg_pn.keys()
244 pref = preferred_versions[p]
245 latest = latest_versions[p]
248 prefstr = pref[0][0] + "-" + pref[0][1]
252 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
255 def buildPackage(item):
258 discriminated = False
260 if not providers.has_key(item):
261 bb.error("Nothing provides %s" % item)
264 all_p = providers[item]
267 if p in __build_cache:
268 bb.debug(1, "already built %s in this run\n" % p)
272 preferred_versions = {}
274 # Collate providers by PN
277 the_data = make.pkgdata[p]
278 pn = bb.data.getVar('PN', the_data, 1)
279 if not pkg_pn.has_key(pn):
283 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
286 for pn in pkg_pn.keys():
290 priority = bbfile_priority[f]
291 if not priorities.has_key(priority):
292 priorities[priority] = []
293 priorities[priority].append(f)
294 p_list = priorities.keys()
295 p_list.sort(lambda a, b: a - b)
298 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
300 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
301 # version. If not, find the latest version provided by an bbfile in the
302 # highest-priority set.
303 for pn in pkg_pn.keys():
304 preferred_file = None
306 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
309 m = re.match('(.*)_(.*)', preferred_v)
311 preferred_v = m.group(1)
312 preferred_r = m.group(2)
314 for file_set in pkg_pn[pn]:
316 the_data = make.pkgdata[f]
317 pv = bb.data.getVar('PV', the_data, 1)
318 pr = bb.data.getVar('PR', the_data, 1)
319 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
321 preferred_ver = (pv, pr)
326 pv_str = '%s-%s' % (preferred_v, preferred_r)
329 if preferred_file is None:
330 bb.note("preferred version %s of %s not available" % (pv_str, pn))
332 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
334 if preferred_file is None:
335 # get highest priority file set
336 files = pkg_pn[pn][0]
341 the_data = make.pkgdata[f]
342 pv = bb.data.getVar('PV', the_data, 1)
343 pr = bb.data.getVar('PR', the_data, 1)
344 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
346 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
350 preferred_file = latest_f
351 preferred_ver = latest
353 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
355 preferred_versions[pn] = (preferred_ver, preferred_file)
356 eligible.append(preferred_file)
359 if p in __build_cache_fail:
360 bb.debug(1, "rejecting already-failed %s" % p)
363 if len(eligible) == 0:
364 bb.error("no eligible providers for %s" % item)
367 # look to see if one of them is already staged, or marked as preferred.
368 # if so, bump it to the head of the queue
370 the_data = make.pkgdata[p]
371 pn = bb.data.getVar('PN', the_data, 1)
372 pv = bb.data.getVar('PV', the_data, 1)
373 pr = bb.data.getVar('PR', the_data, 1)
374 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
375 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
376 if os.path.exists(stamp):
377 (newvers, fn) = preferred_versions[pn]
378 if not fn in eligible:
379 # package was made ineligible by already-failed check
381 oldver = "%s-%s" % (pv, pr)
382 newver = '-'.join(newvers)
383 if (newver != oldver):
384 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
387 if make.options.verbose:
388 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
390 eligible = [fn] + eligible
394 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
396 __preferred[item] = prefervar
398 if __preferred.has_key(item):
400 the_data = make.pkgdata[p]
401 pn = bb.data.getVar('PN', the_data, 1)
402 if __preferred[item] == pn:
403 if make.options.verbose:
404 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
406 eligible = [p] + eligible
410 if len(eligible) > 1 and discriminated == False:
413 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
414 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
415 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
417 # run through the list until we find one that we can build
419 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
420 if try_build(fn, item):
423 bb.note("no buildable providers for %s" % item)
426 def build_depgraph():
431 if bbdebug or progress.p == p: return
433 if os.isatty(sys.stdout.fileno()):
434 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
438 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
440 sys.stdout.write("done.\n")
443 def calc_bbfile_priority(filename):
444 for (regex, pri) in bbfile_config_priorities:
445 if regex.match(filename):
449 # Handle PREFERRED_PROVIDERS
450 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
451 (providee, provider) = p.split(':')
452 if __preferred.has_key(providee) and __preferred[providee] != provider:
453 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
454 __preferred[providee] = provider
456 # Calculate priorities for each file
457 for p in make.pkgdata.keys():
458 bbfile_priority[p] = calc_bbfile_priority(p)
460 n = len(make.pkgdata.keys())
465 bb.debug(1, "BBMAKE building providers hashes")
467 # Build forward and reverse provider hashes
468 # Forward: virtual -> [filenames]
469 # Reverse: PN -> [virtuals]
470 for f in make.pkgdata.keys():
473 pn = bb.data.getVar('PN', d, 1)
474 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
476 if not pn_provides.has_key(pn):
477 pn_provides[pn] = Set()
478 pn_provides[pn] |= provides
480 for provide in provides:
481 if not providers.has_key(provide):
482 providers[provide] = []
483 providers[provide].append(f)
485 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
496 sys.stdout.write("\n")
498 # Build package list for "bbmake world"
499 bb.debug(1, "BBMAKE collating packages for \"world\"")
500 for f in make.pkgdata.keys():
502 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
503 bb.debug(2, "BBMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
506 pn = bb.data.getVar('PN', d, 1)
507 for p in pn_provides[pn]:
508 if p.startswith('virtual/'):
509 bb.debug(2, "BBMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
512 for pf in providers[p]:
513 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
514 bb.debug(2, "BBMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
518 __world_target.add(pn)
520 def myProgressCallback( x, y, f ):
523 if os.isatty(sys.stdout.fileno()):
524 sys.stdout.write("\rNOTE: Parsing .bb files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
528 sys.stdout.write("Parsing .bb files, please wait...")
531 sys.stdout.write("done.")
539 if __name__ == "__main__":
541 if "BBDEBUG" in os.environ:
542 bbdebug = int(os.environ["BBDEBUG"])
544 make.options, args = handle_options( sys.argv )
546 if not make.options.cmd:
547 make.options.cmd = "build"
549 if make.options.cmd in __depcmds:
550 __depcmd=__depcmds[make.options.cmd]
552 __depcmd=make.options.cmd
555 make.cfg = bb.data.init()
558 for f in make.options.file:
560 make.cfg = bb.parse.handle(f, make.cfg)
562 bb.fatal("Unable to open %s" % f)
565 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
567 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
569 if not bb.data.getVar("BUILDNAME", make.cfg):
570 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
572 buildname = bb.data.getVar("BUILDNAME", make.cfg)
574 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
575 __ignored_dependencies = ignore.split()
577 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
579 collection_list = collections.split()
580 for c in collection_list:
581 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
583 bb.error("BBFILE_PATTERN_%s not defined" % c)
585 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
587 bb.error("BBFILE_PRIORITY_%s not defined" % c)
590 cre = re.compile(regex)
592 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
596 bbfile_config_priorities.append((cre, pri))
598 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
602 if not pkgs_to_build:
604 pkgs_to_build.extend(args)
605 if not pkgs_to_build:
606 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
608 pkgs_to_build = bbpkgs.split()
609 if not pkgs_to_build and not make.options.show_versions:
610 print "Nothing to build. Use 'bbmake world' to build everything."
613 __stats["attempt"] = 0
614 __stats["success"] = 0
618 # Import Psyco if available and not disabled
619 if not make.options.disable_psyco:
624 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
626 psyco.bind( make.collect_bbfiles )
628 bb.note("You have disabled Psyco. This decreases performance.")
631 bb.debug(1, "BBMAKE collecting .bb files")
632 make.collect_bbfiles( myProgressCallback )
633 bb.debug(1, "BBMAKE parsing complete")
636 if make.options.parse_only:
637 print "Requested parsing .bb files only. Exiting."
642 if make.options.show_versions:
646 if 'world' in pkgs_to_build:
647 pkgs_to_build.remove('world')
648 for t in __world_target:
649 pkgs_to_build.append(t)
651 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
653 for k in pkgs_to_build:
656 if buildPackage(k) == 0:
659 except bb.build.EventException:
660 bb.error("Build of " + k + " failed")
664 if make.options.abort:
667 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
669 print "Build statistics:"
670 print " Attempted builds: %d" % __stats["attempt"]
671 if __stats["fail"] != 0:
672 print " Failed builds: %d" % __stats["fail"]
673 if __stats["deps"] != 0:
674 print " Dependencies not satisfied: %d" % __stats["deps"]
675 if __stats["fail"] != 0 or __stats["deps"] != 0:
679 except KeyboardInterrupt:
680 print "\nNOTE: KeyboardInterrupt - Build not completed."