2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 import sys, os, getopt, glob, copy, os.path, re
6 sys.path.append('/usr/share/oe')
10 import itertools, optparse
12 parsespin = itertools.cycle( r'|/-\\' )
15 __build_cache_fail = []
21 __world_target = Set()
22 __ignored_dependencies = Set()
23 __depcmds = { "clean": None,
28 oefile_config_priorities = []
32 def handle_options( args ):
33 parser = optparse.OptionParser( version = "OpenEmbedded Build Infrastructure Core version %s, %%prog version %s" % ( oe.__version__, __version__ ),
34 usage = """%prog [options] [package ...]
36 Builds specified packages, expecting that the .oe files
37 it has to work from are in OEFILES
38 Default packages are all packages in OEFILES.
39 Default OEFILES are the .oe files in the current directory.""" )
41 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
42 action = "store_false", dest = "abort", default = True )
44 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
45 action = "store_true", dest = "force", default = False )
48 parser.add_option( "-c", "--cmd", help = "specify command to pass to oebuild. Valid commands are "
49 "'fetch' (fetch all sources), "
50 "'unpack' (unpack the sources), "
51 "'patch' (apply the patches), "
52 "'configure' (configure the source tree), "
53 "'compile' (compile the source tree), "
54 "'stage' (install libraries and headers needed for subsequent packages), "
55 "'install' (install libraries and executables), and"
56 "'package' (package files into the selected package format)",
57 action = "store", dest = "cmd", default = "build" )
59 parser.add_option( "-r", "--read", help = "read the specified file before oe.conf",
60 action = "append", dest = "file", default = [] )
62 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
63 action = "store_true", dest = "verbose", default = False )
65 parser.add_option( "-n", "--dry-run", help = "don't call oebuild, just go through the motions",
66 action = "store_true", dest = "dry_run", default = False )
68 parser.add_option( "-p", "--parse-only", help = "quit after parsing the OE files (developers only)",
69 action = "store_true", dest = "parse_only", default = False )
71 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
72 action = "store_true", dest = "disable_psyco", default = False )
74 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
75 action = "store_true", dest = "show_versions", default = False )
77 options, args = parser.parse_args( args )
78 return options, args[1:]
80 def try_build(fn, virtual):
81 if fn in __building_list:
82 oe.error("%s depends on itself (eventually)" % fn)
83 oe.error("upwards chain is: %s" % (" -> ".join(__build_path)))
86 __building_list.append(fn)
88 the_data = make.pkgdata[fn]
89 item = oe.data.getVar('PN', the_data, 1)
90 pathstr = "%s (%s)" % (item, virtual)
91 __build_path.append(pathstr)
93 depends_list = (oe.data.getVar('DEPENDS', the_data, 1) or "").split()
94 if make.options.verbose:
95 oe.note("current path: %s" % (" -> ".join(__build_path)))
96 oe.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
102 oldcmd = make.options.cmd
103 make.options.cmd = __depcmd
105 for d in depends_list:
106 if d in __ignored_dependencies:
110 if buildPackage(d) == 0:
111 oe.error("dependency %s (for %s) not satisfied" % (d,item))
113 if make.options.abort:
117 make.options.cmd = oldcmd
123 oe.event.fire(oe.event.PkgStarted(item, make.pkgdata[fn]))
125 __stats["attempt"] += 1
126 if not make.options.dry_run:
127 oe.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
128 oe.event.fire(oe.event.PkgSucceeded(item, make.pkgdata[fn]))
129 __build_cache.append(fn)
131 except oe.build.FuncFailed:
133 oe.error("task stack execution failed")
134 oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
135 __build_cache_fail.append(fn)
137 except oe.build.EventException:
139 (type, value, traceback) = sys.exc_info()
141 oe.error("%s event exception, aborting" % oe.event.getName(e))
142 oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
143 __build_cache_fail.append(fn)
146 __building_list.remove(fn)
147 __build_path.remove(pathstr)
151 preferred_versions = {}
154 for p in make.pkgdata.keys():
155 pn = oe.data.getVar('PN', make.pkgdata[p], 1)
156 if not pkg_pn.has_key(pn):
161 for pn in pkg_pn.keys():
165 priority = oefile_priority[f]
166 if not priorities.has_key(priority):
167 priorities[priority] = []
168 priorities[priority].append(f)
169 p_list = priorities.keys()
170 p_list.sort(lambda a, b: a - b)
173 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
175 # If there is a PREFERRED_VERSION, find the highest-priority oefile providing that
176 # version. If not, find the latest version provided by an oefile in the
177 # highest-priority set.
178 for pn in pkg_pn.keys():
179 preferred_file = None
181 preferred_v = oe.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
184 m = re.match('(.*)_(.*)', preferred_v)
186 preferred_v = m.group(1)
187 preferred_r = m.group(2)
189 for file_set in pkg_pn[pn]:
191 the_data = make.pkgdata[f]
192 pv = oe.data.getVar('PV', the_data, 1)
193 pr = oe.data.getVar('PR', the_data, 1)
194 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
196 preferred_ver = (pv, pr)
201 pv_str = '%s-%s' % (preferred_v, preferred_r)
204 if preferred_file is None:
205 oe.note("preferred version %s of %s not available" % (pv_str, pn))
207 oe.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
209 # get highest priority file set
210 files = pkg_pn[pn][0]
215 the_data = make.pkgdata[f]
216 pv = oe.data.getVar('PV', the_data, 1)
217 pr = oe.data.getVar('PR', the_data, 1)
218 dp = int(oe.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
220 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
224 if preferred_file is None:
225 preferred_file = latest_f
226 preferred_ver = latest
228 preferred_versions[pn] = (preferred_ver, preferred_file)
229 latest_versions[pn] = (latest, latest_f)
231 pkg_list = pkg_pn.keys()
235 pref = preferred_versions[p]
236 latest = latest_versions[p]
239 prefstr = pref[0][0] + "-" + pref[0][1]
243 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
246 def buildPackage(item):
249 discriminated = False
251 if not providers.has_key(item):
252 oe.error("Nothing provides %s" % item)
255 all_p = providers[item]
258 if p in __build_cache:
259 oe.debug(1, "already built %s in this run\n" % p)
263 preferred_versions = {}
265 # Collate providers by PN
268 the_data = make.pkgdata[p]
269 pn = oe.data.getVar('PN', the_data, 1)
270 if not pkg_pn.has_key(pn):
274 oe.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
277 for pn in pkg_pn.keys():
281 priority = oefile_priority[f]
282 if not priorities.has_key(priority):
283 priorities[priority] = []
284 priorities[priority].append(f)
285 p_list = priorities.keys()
286 p_list.sort(lambda a, b: a - b)
289 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
291 # If there is a PREFERRED_VERSION, find the highest-priority oefile providing that
292 # version. If not, find the latest version provided by an oefile in the
293 # highest-priority set.
294 for pn in pkg_pn.keys():
295 preferred_file = None
297 preferred_v = oe.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
300 m = re.match('(.*)_(.*)', preferred_v)
302 preferred_v = m.group(1)
303 preferred_r = m.group(2)
305 for file_set in pkg_pn[pn]:
307 the_data = make.pkgdata[f]
308 pv = oe.data.getVar('PV', the_data, 1)
309 pr = oe.data.getVar('PR', the_data, 1)
310 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
312 preferred_ver = (pv, pr)
317 pv_str = '%s-%s' % (preferred_v, preferred_r)
320 if preferred_file is None:
321 oe.note("preferred version %s of %s not available" % (pv_str, pn))
323 oe.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
325 if preferred_file is None:
326 # get highest priority file set
327 files = pkg_pn[pn][0]
332 the_data = make.pkgdata[f]
333 pv = oe.data.getVar('PV', the_data, 1)
334 pr = oe.data.getVar('PR', the_data, 1)
335 dp = int(oe.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
337 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
341 preferred_file = latest_f
342 preferred_ver = latest
344 oe.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
346 preferred_versions[pn] = (preferred_ver, preferred_file)
347 eligible.append(preferred_file)
350 if p in __build_cache_fail:
351 oe.debug(1, "rejecting already-failed %s" % p)
354 if len(eligible) == 0:
355 oe.error("no eligible providers for %s" % item)
358 # look to see if one of them is already staged, or marked as preferred.
359 # if so, bump it to the head of the queue
361 the_data = make.pkgdata[p]
362 pn = oe.data.getVar('PN', the_data, 1)
363 pv = oe.data.getVar('PV', the_data, 1)
364 pr = oe.data.getVar('PR', the_data, 1)
365 tmpdir = oe.data.getVar('TMPDIR', the_data, 1)
366 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
367 if os.path.exists(stamp):
368 (newvers, fn) = preferred_versions[pn]
369 if not fn in eligible:
370 # package was made ineligible by already-failed check
372 oldver = "%s-%s" % (pv, pr)
373 newver = '-'.join(newvers)
374 if (newver != oldver):
375 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
378 if make.options.verbose:
379 oe.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
381 eligible = [fn] + eligible
385 prefervar = oe.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
387 __preferred[item] = prefervar
389 if __preferred.has_key(item):
391 the_data = make.pkgdata[p]
392 pn = oe.data.getVar('PN', the_data, 1)
393 if __preferred[item] == pn:
394 if make.options.verbose:
395 oe.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
397 eligible = [p] + eligible
401 if len(eligible) > 1 and discriminated == False:
404 providers_list.append(oe.data.getVar('PN', make.pkgdata[fn], 1))
405 oe.note("multiple providers are available (%s);" % ", ".join(providers_list))
406 oe.note("consider defining PREFERRED_PROVIDER_%s" % item)
408 # run through the list until we find one that we can build
410 oe.debug(2, "selecting %s to satisfy %s" % (fn, item))
411 if try_build(fn, item):
414 oe.note("no buildable providers for %s" % item)
417 def build_depgraph():
422 if oedebug or progress.p == p: return
424 if os.isatty(sys.stdout.fileno()):
425 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
429 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
431 sys.stdout.write("done.\n")
434 def calc_oefile_priority(filename):
435 for (regex, pri) in oefile_config_priorities:
436 if regex.match(filename):
440 # Handle PREFERRED_PROVIDERS
441 for p in (oe.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
442 (providee, provider) = p.split(':')
443 if __preferred.has_key(providee) and __preferred[providee] != provider:
444 oe.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
445 __preferred[providee] = provider
447 # Calculate priorities for each file
448 for p in make.pkgdata.keys():
449 oefile_priority[p] = calc_oefile_priority(p)
451 n = len(make.pkgdata.keys())
456 oe.debug(1, "OEMAKE building providers hashes")
458 # Build forward and reverse provider hashes
459 # Forward: virtual -> [filenames]
460 # Reverse: PN -> [virtuals]
461 for f in make.pkgdata.keys():
464 pn = oe.data.getVar('PN', d, 1)
465 provides = Set([pn] + (oe.data.getVar("PROVIDES", d, 1) or "").split())
467 if not pn_provides.has_key(pn):
468 pn_provides[pn] = Set()
469 pn_provides[pn] |= provides
471 for provide in provides:
472 if not providers.has_key(provide):
473 providers[provide] = []
474 providers[provide].append(f)
476 deps = (oe.data.getVar("DEPENDS", d, 1) or "").split()
487 sys.stdout.write("\n")
489 # Build package list for "oemake world"
490 oe.debug(1, "OEMAKE collating packages for \"world\"")
491 for f in make.pkgdata.keys():
493 if oe.data.getVar('BROKEN', d, 1):
496 pn = oe.data.getVar('PN', d, 1)
497 for p in pn_provides[pn]:
498 if p in all_depends or p.startswith('virtual/'):
502 __world_target.add(pn)
504 def myProgressCallback( x, y, f ):
507 if os.isatty(sys.stdout.fileno()):
508 sys.stdout.write("\rNOTE: Parsing .oe files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
512 sys.stdout.write("Parsing .oe files, please wait...")
515 sys.stdout.write("done.")
523 if __name__ == "__main__":
525 if "OEDEBUG" in os.environ:
526 oedebug = int(os.environ["OEDEBUG"])
528 make.options, args = handle_options( sys.argv )
530 if not make.options.cmd:
531 make.options.cmd = "build"
533 if make.options.cmd in __depcmds:
534 __depcmd=__depcmds[make.options.cmd]
536 __depcmd=make.options.cmd
539 make.cfg = oe.data.init()
542 for f in make.options.file:
544 make.cfg = oe.parse.handle(f, make.cfg)
546 oe.fatal("Unable to open %s" % f)
549 make.cfg = oe.parse.handle("conf/oe.conf", make.cfg)
551 oe.fatal("Unable to open oe.conf")
553 if not oe.data.getVar("BUILDNAME", make.cfg):
554 oe.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
556 buildname = oe.data.getVar("BUILDNAME", make.cfg)
558 ignore = oe.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
559 __ignored_dependencies = ignore.split()
561 collections = oe.data.getVar("OEFILE_COLLECTIONS", make.cfg, 1)
563 collection_list = collections.split()
564 for c in collection_list:
565 regex = oe.data.getVar("OEFILE_PATTERN_%s" % c, make.cfg, 1)
567 oe.error("OEFILE_PATTERN_%s not defined" % c)
569 priority = oe.data.getVar("OEFILE_PRIORITY_%s" % c, make.cfg, 1)
571 oe.error("OEFILE_PRIORITY_%s not defined" % c)
574 cre = re.compile(regex)
576 oe.error("OEFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
580 oefile_config_priorities.append((cre, pri))
582 oe.error("invalid value for OEFILE_PRIORITY_%s: \"%s\"" % (c, priority))
586 if not pkgs_to_build:
588 pkgs_to_build.extend(args)
589 if not pkgs_to_build:
590 oepkgs = oe.data.getVar('OEPKGS', make.cfg, 1)
592 pkgs_to_build = oepkgs.split()
593 if not pkgs_to_build and not make.options.show_versions:
594 print "Nothing to build. Use 'oemake world' to build everything."
597 __stats["attempt"] = 0
598 __stats["success"] = 0
602 # Import Psyco if available and not disabled
603 if not make.options.disable_psyco:
608 oe.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
610 psyco.bind( make.collect_oefiles )
612 oe.note("You have disabled Psyco. This decreases performance.")
615 oe.debug(1, "OEMAKE collecting .oe files")
616 make.collect_oefiles( myProgressCallback )
617 oe.debug(1, "OEMAKE parsing complete")
620 if make.options.parse_only:
621 print "Requested parsing .oe files only. Exiting."
626 if make.options.show_versions:
630 if 'world' in pkgs_to_build:
631 pkgs_to_build.remove('world')
632 for t in __world_target:
633 pkgs_to_build.append(t)
635 oe.event.fire(oe.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
637 for k in pkgs_to_build:
640 if buildPackage(k) == 0:
643 except oe.build.EventException:
644 oe.error("Build of " + k + " failed")
648 if make.options.abort:
651 oe.event.fire(oe.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
653 print "Build statistics:"
654 print " Attempted builds: %d" % __stats["attempt"]
655 if __stats["fail"] != 0:
656 print " Failed builds: %d" % __stats["fail"]
657 if __stats["deps"] != 0:
658 print " Dependencies not satisfied: %d" % __stats["deps"]
659 if __stats["fail"] != 0 or __stats["deps"] != 0:
663 except KeyboardInterrupt:
664 print "\nNOTE: KeyboardInterrupt - Build not completed."