avoid python crash if latest version of a package becomes ineligible - patch courtesy...
[bitbake.git] / bin / oemake
1 #!/usr/bin/env python
2
3 import sys, os, getopt, glob, copy, os.path, re
4 sys.path.append('/usr/share/oe')
5 import oe
6 from oe import make
7 from sets import Set
8
9 try:
10     import itertools
11 except ImportError: # itertools appears in Python 2.3
12     from utils import itertools
13 try:
14     import optparse
15 except ImportError: # optparse appears in Python 2.3
16     from utils import optparse
17 parsespin = itertools.cycle( r'|/-\-' )
18
19 __version__ = 1.1
20 __build_cache_fail = []
21 __build_cache = []
22 __building_list = []
23 __build_path = []
24
25 __preferred = {}
26 __world_target = Set()
27 __ignored_dependencies = Set()
28
29 __stats = {}
30
31 def handle_options( args ):
32         parser = optparse.OptionParser( version = "OpenEmbedded Build Infrastructure Core version %s, %%prog version %s" % ( oe.__version__, __version__ ),
33         usage = """%prog [options] [package ...]
34
35 Builds specified packages, expecting that the .oe files
36 it has to work from are in OEFILES
37 Default packages are all packages in OEFILES.
38 Default OEFILES are the .oe files in the current directory.""" )
39
40         parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
41                            action = "store_false", dest = "abort", default = True )
42
43         parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
44                            action = "store_true", dest = "force", default = False )
45
46
47         parser.add_option( "-c", "--cmd", help = "specify command to pass to oebuild",
48                            action = "store", dest = "cmd", default = "build" )
49
50         parser.add_option( "-r", "--read", help = "read the specified file before oe.conf",
51                            action = "append", dest = "file", default = [] )
52
53         parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
54                            action = "store_true", dest = "verbose", default = False )
55
56         parser.add_option( "-n", "--dry-run", help = "don't call oebuild, just go through the motions",
57                            action = "store_true", dest = "dry_run", default = False )
58
59         parser.add_option( "-p", "--parse-only", help = "quit after parsing the OE files (developers only)",
60                            action = "store_true", dest = "parse_only", default = False )
61
62         parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
63                            action = "store_true", dest = "disable_psyco", default = False )
64
65         options, args = parser.parse_args( args )
66         return options, args[1:]
67
68 def try_build(fn, virtual):
69         if fn in __building_list:
70                 oe.error("%s depends on itself (eventually)" % fn)
71                 oe.error("upwards chain is: %s" % (" -> ".join(__build_path)))
72                 return False
73
74         __building_list.append(fn)
75
76         the_data = make.pkgdata[fn]
77         item = oe.data.getVar('PN', the_data, 1)
78         pathstr = "%s (%s)" % (item, virtual)
79         __build_path.append(pathstr)
80
81         depends_list = (oe.data.getVar('DEPENDS', the_data, 1) or "").split()
82         if make.options.verbose:
83                 oe.note("current path: %s" % (" -> ".join(__build_path)))
84                 oe.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
85
86         try:
87                 failed = False
88                 
89                 for d in depends_list:
90                         if d in __ignored_dependencies:
91                                 continue
92                         if buildPackage(d) == 0:
93                                 failed = True
94                                 if make.options.abort:
95                                         break
96
97                 if failed:
98                         oe.error("dependency %s (for %s) not satisfied" % (d,item))
99                         __stats["deps"] += 1
100                         return False
101
102                 command = make.options.cmd
103                 oe.debug(1, "oebuild %s %s" % (command, fn))
104                 oe.event.fire(oe.event.PkgStarted(item, make.pkgdata[fn]))
105                 try:
106                         __stats["attempt"] += 1
107                         if not make.options.dry_run:
108                                 oe.build.exec_task('do_%s' % command, make.pkgdata[fn])
109                         oe.event.fire(oe.event.PkgSucceeded(item, make.pkgdata[fn]))
110                         __build_cache.append(fn)
111                         return True
112                 except oe.build.FuncFailed:
113                         __stats["fail"] += 1
114                         oe.error("task stack execution failed")
115                         oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
116                         __build_cache_fail.append(fn)
117                         return False
118                 except oe.build.EventException:
119                         __stats["fail"] += 1
120                         (type, value, traceback) = sys.exc_info()
121                         e = value.event
122                         oe.error("%s event exception, aborting" % oe.event.getName(e))
123                         oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
124                         __build_cache_fail.append(fn)
125                         return False
126         finally:
127                 __building_list.remove(fn)
128                 __build_path.remove(pathstr)
129                 
130
131 def buildPackage(item):
132         fn = None
133         
134         if not providers.has_key(item):
135                 oe.error("Nothing provides %s" % item)
136                 return 0
137
138         all_p = providers[item]
139
140         for p in all_p:
141                 if p in __build_cache:
142                         return 1
143
144         versions = {}
145         for p in all_p:
146                 the_data = make.pkgdata[p]
147                 pn = oe.data.getVar('PN', the_data, 1)
148                 pv = oe.data.getVar('PV', the_data, 1)
149                 pr = oe.data.getVar('PR', the_data, 1)
150                 if not versions.has_key(pn):
151                         versions[pn] = []
152                 versions[pn].append(((pv, pr), p))
153
154         # find the latest version of each provider
155         preferred_versions = {}
156         for p in versions.keys():
157                 latest = None
158                 latest_f = None
159                 for (v, _fn) in versions[p]:
160                         if (latest is None) or (make.vercmp(latest, v) < 0):
161                                 latest = v
162                                 latest_f = _fn
163                 preferred_versions[p] = (latest, latest_f)
164             
165         # build a new list with just the latest version of everything
166         eligible = []
167         for p in preferred_versions.keys():
168                 (v, f) = preferred_versions[p]
169                 eligible.append(f)
170
171         for p in eligible:
172                 if p in __build_cache_fail:
173                         oe.debug(1, "rejecting already-failed %s" % p)
174                         eligible.remove(p)
175
176         if len(eligible) == 0:
177                 oe.error("no eligible providers for %s" % item)
178                 return 0
179
180         # look to see if one of them is already staged, or marked as preferred.
181         # if so, bump it to the head of the queue
182         for p in all_p:
183                 the_data = make.pkgdata[p]
184                 pn = oe.data.getVar('PN', the_data, 1)
185                 pv = oe.data.getVar('PV', the_data, 1)
186                 pr = oe.data.getVar('PR', the_data, 1)
187                 tmpdir = oe.data.getVar('TMPDIR', the_data, 1)
188                 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
189                 if os.path.exists(stamp):
190                         (newvers, fn) = preferred_versions[pn]
191                         if not fn in eligible:
192                                 # package was made ineligible by already-failed check
193                                 continue
194                         oldver = "%s-%s" % (pv, pr)
195                         newver = '-'.join(newvers)
196                         if (newver != oldver):
197                                 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
198                         else:
199                                 extra_chat = ""
200                         if make.options.verbose:
201                                 oe.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
202                         eligible.remove(fn)
203                         eligible = [fn] + eligible
204                         break
205
206         if __preferred.has_key(item):
207                 for p in eligible:
208                         the_data = make.pkgdata[p]
209                         pn = oe.data.getVar('PN', the_data, 1)
210                         if __preferred[item] == pn:
211                                 if make.options.verbose:
212                                         oe.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
213                                 eligible.remove(p)
214                                 eligible = [p] + eligible
215                                 break
216                 
217         # run through the list until we find one that we can build
218         for fn in eligible:
219                 oe.debug(2, "selecting %s to satisfy %s" % (fn, item))
220                 if try_build(fn, item):
221                         return 1
222
223         oe.note("no buildable providers for %s" % item)
224         return 0
225
226
227 def build_depgraph():
228     all_depends = Set()
229     pn_provides = {}
230     
231     for f in make.pkgdata.keys():
232         d = make.pkgdata[f]
233         
234         pn = oe.data.getVar('PN', d, 1)
235
236         deps = (oe.data.getVar("DEPENDS", d, 1) or "").split()
237         provides = Set([pn] + (oe.data.getVar("PROVIDES", d, 1) or "").split())
238
239         for dep in deps:
240             all_depends.add(dep)
241
242         if not pn_provides.has_key(pn):
243             pn_provides[pn] = Set()
244         pn_provides[pn] |= provides
245         
246         for provide in provides:
247             if not providers.has_key(provide):
248                 providers[provide] = []
249             providers[provide].append(f)
250
251         for p in (oe.data.getVar('PREFERRED_PROVIDERS', d, 1) or "").split():
252             (providee, provider) = p.split(':')
253             if __preferred.has_key(providee) and __preferred[providee] != provider:
254                 oe.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
255             __preferred[providee] = provider
256
257     for f in make.pkgdata.keys():
258         d = make.pkgdata[f]
259         terminal = True
260         pn = oe.data.getVar('PN', d, 1)
261         for p in pn_provides[pn]:
262             if p in all_depends or p.startswith('virtual/'):
263                 terminal = False
264                 break
265         if terminal:
266             __world_target.add(pn)
267
268 def myProgressCallback( x, y, f ):
269         sys.stdout.write("\rNOTE: Parsing .oe files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
270         sys.stdout.flush()
271
272 #
273 # main
274 #
275
276 if __name__ == "__main__":
277
278     make.options, args = handle_options( sys.argv )
279
280     _depcmds = { "clean": None,
281                 "mrproper": None,
282                 "build": "stage" }
283
284     if not make.options.cmd:
285             make.options.cmd = "build"
286
287     if make.options.cmd in _depcmds:
288             depcmd=_depcmds[make.options.cmd]
289     else:
290             depcmd=make.options.cmd
291
292     make.pkgdata = {}
293     make.cfg = {}
294     providers = {}
295
296     for f in make.options.file:
297             try:
298                     make.cfg = oe.parse.handle(f, make.cfg)
299             except IOError:
300                     oe.fatal("Unable to open %s" % f)
301
302     try:
303             make.cfg = oe.parse.handle("conf/oe.conf", make.cfg)
304     except IOError:
305             oe.fatal("Unable to open oe.conf")
306
307     if not oe.data.getVar("BUILDNAME", make.cfg):
308             oe.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
309
310     buildname = oe.data.getVar("BUILDNAME", make.cfg)
311
312     ignore = oe.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
313     __ignored_dependencies = ignore.split()
314
315     pkgs_to_build = None
316     if args:
317             if not pkgs_to_build:
318                     pkgs_to_build = []
319             pkgs_to_build.extend(args)
320     if not pkgs_to_build:
321             oepkgs = oe.data.getVar('OEPKGS', make.cfg, 1)
322             if oepkgs:
323                     pkgs_to_build = oepkgs.split()
324     if not pkgs_to_build:
325             print "Nothing to build. Use 'oemake world' to build everything."
326             sys.exit(0)
327
328     __stats["attempt"] = 0
329     __stats["success"] = 0
330     __stats["fail"] = 0
331     __stats["deps"] = 0
332
333     # Import Psyco if available and not disabled
334     if not make.options.disable_psyco:
335         try:
336             import psyco
337         except ImportError:
338             print "NOTE: Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance."
339         else:
340             psyco.bind( make.collect_oefiles )
341     else:
342         print "NOTE: You have disabled Psyco. This decreases performance."
343
344     try:
345             make.collect_oefiles( myProgressCallback )
346             print
347             if make.options.parse_only:
348                     print "Requested parsing .oe files only.  Exiting."
349                     sys.exit(0)
350             build_depgraph()
351
352             if 'world' in pkgs_to_build:
353                     pkgs_to_build.remove('world')
354                     for t in __world_target:
355                             pkgs_to_build.append(t)
356
357             oe.event.fire(oe.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
358
359             for k in pkgs_to_build:
360                     if buildPackage(k) == 0:
361                             oe.error("Build of " + k + " failed")
362                             if make.options.abort:
363                                     sys.exit(1)
364
365             oe.event.fire(oe.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
366
367             print "Build statistics:"
368             print "  Attempted builds: %d" % __stats["attempt"]
369             if __stats["fail"] != 0:
370                     print "  Failed builds: %d" % __stats["fail"]
371             if __stats["deps"] != 0:
372                     print "  Dependencies not satisfied: %d" % __stats["deps"]
373             if __stats["fail"] != 0 or __stats["deps"] != 0:
374                     sys.exit(1)
375             sys.exit(0)
376
377     except KeyboardInterrupt:
378             print "\nNOTE: KeyboardInterrupt - Build not completed."