another try at importing ASSUME_PROVIDED patch - patch courtesy pb_.
[bitbake.git] / bin / oemake
1 #!/usr/bin/env python
2
3 import sys, os, getopt, glob, copy, os.path, re
4 sys.path.append('/usr/share/oe')
5 import oe
6 from oe import make
7 from sets import Set
8
9 try:
10     import itertools
11 except ImportError: # itertools appears in Python 2.3
12     from utils import itertools
13 try:
14     import optparse
15 except ImportError: # optparse appears in Python 2.3
16     from utils import optparse
17 parsespin = itertools.cycle( r'|/-\-' )
18
19 __version__ = 1.1
20 __build_cache_fail = []
21 __build_cache = []
22 __building_list = []
23 __build_path = []
24
25 __preferred = {}
26 __world_target = Set()
27 __ignored_dependencies = Set()
28
29 __stats = {}
30
31 def handle_options( args ):
32         parser = optparse.OptionParser( version = "OpenEmbedded Build Infrastructure Core version %s, %%prog version %s" % ( oe.__version__, __version__ ),
33         usage = """%prog [options] [package ...]
34
35 Builds specified packages, expecting that the .oe files
36 it has to work from are in OEFILES
37 Default packages are all packages in OEFILES.
38 Default OEFILES are the .oe files in the current directory.""" )
39
40         parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
41                            action = "store_false", dest = "abort", default = True )
42
43         parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
44                            action = "store_true", dest = "force", default = False )
45
46
47         parser.add_option( "-c", "--cmd", help = "specify command to pass to oebuild",
48                            action = "store", dest = "cmd", default = "build" )
49
50         parser.add_option( "-r", "--read", help = "read the specified file before oe.conf",
51                            action = "append", dest = "file", default = [] )
52
53         parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
54                            action = "store_true", dest = "verbose", default = False )
55
56         parser.add_option( "-n", "--dry-run", help = "don't call oebuild, just go through the motions",
57                            action = "store_true", dest = "dry_run", default = False )
58
59         parser.add_option( "-p", "--parse-only", help = "quit after parsing the OE files (developers only)",
60                            action = "store_true", dest = "parse_only", default = False )
61
62         parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
63                            action = "store_true", dest = "disable_psyco", default = False )
64
65         options, args = parser.parse_args( args )
66         return options, args[1:]
67
68 def try_build(fn, virtual):
69         if fn in __building_list:
70                 oe.error("%s depends on itself (eventually)" % fn)
71                 oe.error("upwards chain is: %s" % (" -> ".join(__build_path)))
72                 return False
73
74         __building_list.append(fn)
75
76         the_data = make.pkgdata[fn]
77         item = oe.data.getVar('PN', the_data, 1)
78         pathstr = "%s (%s)" % (item, virtual)
79         __build_path.append(pathstr)
80
81         depends_list = (oe.data.getVar('DEPENDS', the_data, 1) or "").split()
82         if make.options.verbose:
83                 oe.note("current path: %s" % (" -> ".join(__build_path)))
84                 oe.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
85
86         try:
87                 failed = False
88                 
89                 for d in depends_list:
90                         if d in __ignored_dependencies:
91                                 continue
92                         if buildPackage(d) == 0:
93                                 failed = True
94                                 if make.options.abort:
95                                         break
96
97                 if failed:
98                         oe.error("dependency %s (for %s) not satisfied" % (d,item))
99                         __stats["deps"] += 1
100                         return False
101
102                 command = make.options.cmd
103                 oe.debug(1, "oebuild %s %s" % (command, fn))
104                 oe.event.fire(oe.event.PkgStarted(item, make.pkgdata[fn]))
105                 try:
106                         __stats["attempt"] += 1
107                         if not make.options.dry_run:
108                                 oe.build.exec_task('do_%s' % command, make.pkgdata[fn])
109                         oe.event.fire(oe.event.PkgSucceeded(item, make.pkgdata[fn]))
110                         __build_cache.append(fn)
111                         return True
112                 except oe.build.FuncFailed:
113                         __stats["fail"] += 1
114                         oe.error("task stack execution failed")
115                         oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
116                         __build_cache_fail.append(fn)
117                         return False
118                 except oe.build.EventException:
119                         __stats["fail"] += 1
120                         (type, value, traceback) = sys.exc_info()
121                         e = value.event
122                         oe.error("%s event exception, aborting" % oe.event.getName(e))
123                         oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
124                         __build_cache_fail.append(fn)
125                         return False
126         finally:
127                 __building_list.remove(fn)
128                 __build_path.remove(pathstr)
129                 
130
131 def buildPackage(item):
132         fn = None
133         
134         if not providers.has_key(item):
135                 oe.error("Nothing provides %s" % item)
136                 return 0
137
138         all_p = providers[item]
139
140         for p in all_p:
141                 if p in __build_cache:
142                         return 1
143
144         versions = {}
145         for p in all_p:
146                 the_data = make.pkgdata[p]
147                 pn = oe.data.getVar('PN', the_data, 1)
148                 pv = oe.data.getVar('PV', the_data, 1)
149                 pr = oe.data.getVar('PR', the_data, 1)
150                 if not versions.has_key(pn):
151                         versions[pn] = []
152                 versions[pn].append(((pv, pr), p))
153
154         # find the latest version of each provider
155         preferred_versions = {}
156         for p in versions.keys():
157                 latest = None
158                 latest_f = None
159                 for (v, _fn) in versions[p]:
160                         if (latest is None) or (make.vercmp(latest, v) < 0):
161                                 latest = v
162                                 latest_f = _fn
163                 preferred_versions[p] = (latest, latest_f)
164             
165         # build a new list with just the latest version of everything
166         eligible = []
167         for p in preferred_versions.keys():
168                 (v, f) = preferred_versions[p]
169                 eligible.append(f)
170
171         for p in eligible:
172                 if p in __build_cache_fail:
173                         oe.debug(1, "rejecting already-failed %s" % p)
174                         eligible.remove(p)
175
176         if len(eligible) == 0:
177                 oe.error("no eligible providers for %s" % item)
178                 return 0
179
180         # look to see if one of them is already staged, or marked as preferred.
181         # if so, bump it to the head of the queue
182         for p in all_p:
183                 the_data = make.pkgdata[p]
184                 pn = oe.data.getVar('PN', the_data, 1)
185                 pv = oe.data.getVar('PV', the_data, 1)
186                 pr = oe.data.getVar('PR', the_data, 1)
187                 tmpdir = oe.data.getVar('TMPDIR', the_data, 1)
188                 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
189                 if os.path.exists(stamp):
190                         (newvers, fn) = preferred_versions[pn]
191                         oldver = "%s-%s" % (pv, pr)
192                         newver = '-'.join(newvers)
193                         if (newver != oldver):
194                                 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
195                         else:
196                                 extra_chat = ""
197                         if make.options.verbose:
198                                 oe.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
199                         eligible.remove(fn)
200                         eligible = [fn] + eligible
201                         break
202
203         if __preferred.has_key(item):
204                 for p in eligible:
205                         the_data = make.pkgdata[p]
206                         pn = oe.data.getVar('PN', the_data, 1)
207                         if __preferred[item] == pn:
208                                 if make.options.verbose:
209                                         oe.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
210                                 eligible.remove(p)
211                                 eligible = [p] + eligible
212                                 break
213                 
214         # run through the list until we find one that we can build
215         for fn in eligible:
216                 oe.debug(2, "selecting %s to satisfy %s" % (fn, item))
217                 if try_build(fn, item):
218                         return 1
219
220         oe.note("no buildable providers for %s" % item)
221         return 0
222
223
224 def build_depgraph():
225     all_depends = Set()
226     pn_provides = {}
227     
228     for f in make.pkgdata.keys():
229         d = make.pkgdata[f]
230         
231         pn = oe.data.getVar('PN', d, 1)
232
233         deps = (oe.data.getVar("DEPENDS", d, 1) or "").split()
234         provides = Set([pn] + (oe.data.getVar("PROVIDES", d, 1) or "").split())
235
236         for dep in deps:
237             all_depends.add(dep)
238
239         if not pn_provides.has_key(pn):
240             pn_provides[pn] = Set()
241         pn_provides[pn] |= provides
242         
243         for provide in provides:
244             if not providers.has_key(provide):
245                 providers[provide] = []
246             providers[provide].append(f)
247
248         for p in (oe.data.getVar('PREFERRED_PROVIDERS', d, 1) or "").split():
249             (providee, provider) = p.split(':')
250             if __preferred.has_key(providee) and __preferred[providee] != provider:
251                 oe.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
252             __preferred[providee] = provider
253
254     for f in make.pkgdata.keys():
255         d = make.pkgdata[f]
256         terminal = True
257         pn = oe.data.getVar('PN', d, 1)
258         for p in pn_provides[pn]:
259             if p in all_depends or p.startswith('virtual/'):
260                 terminal = False
261                 break
262         if terminal:
263             __world_target.add(pn)
264
265 def myProgressCallback( x, y, f ):
266         sys.stdout.write("\rNOTE: Parsing .oe files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
267         sys.stdout.flush()
268
269 #
270 # main
271 #
272
273 if __name__ == "__main__":
274
275     make.options, args = handle_options( sys.argv )
276
277     _depcmds = { "clean": None,
278                 "mrproper": None,
279                 "build": "stage" }
280
281     if not make.options.cmd:
282             make.options.cmd = "build"
283
284     if make.options.cmd in _depcmds:
285             depcmd=_depcmds[make.options.cmd]
286     else:
287             depcmd=make.options.cmd
288
289     make.pkgdata = {}
290     make.cfg = {}
291     providers = {}
292
293     for f in make.options.file:
294             try:
295                     make.cfg = oe.parse.handle(f, make.cfg)
296             except IOError:
297                     oe.fatal("Unable to open %s" % f)
298
299     try:
300             make.cfg = oe.parse.handle("conf/oe.conf", make.cfg)
301     except IOError:
302             oe.fatal("Unable to open oe.conf")
303
304     if not oe.data.getVar("BUILDNAME", make.cfg):
305             oe.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
306
307     buildname = oe.data.getVar("BUILDNAME", make.cfg)
308
309     ignore = oe.data.getVar("ASSUME_PROVIDED", make.cfg) or ""
310     __ignored_dependencies = ignore.split()
311
312     pkgs_to_build = None
313     if args:
314             if not pkgs_to_build:
315                     pkgs_to_build = []
316             pkgs_to_build.extend(args)
317     if not pkgs_to_build:
318             oepkgs = oe.data.getVar('OEPKGS', make.cfg, 1)
319             if oepkgs:
320                     pkgs_to_build = oepkgs.split()
321     if not pkgs_to_build:
322             print "Nothing to build. Use 'oemake world' to build everything."
323             sys.exit(0)
324
325     __stats["attempt"] = 0
326     __stats["success"] = 0
327     __stats["fail"] = 0
328     __stats["deps"] = 0
329
330     # Import Psyco if available and not disabled
331     if not make.options.disable_psyco:
332         try:
333             import psyco
334         except ImportError:
335             print "NOTE: Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance."
336         else:
337             psyco.bind( make.collect_oefiles )
338     else:
339         print "NOTE: You have disabled Psyco. This decreases performance."
340
341     try:
342             make.collect_oefiles( myProgressCallback )
343             print
344             if make.options.parse_only:
345                     print "Requested parsing .oe files only.  Exiting."
346                     sys.exit(0)
347             build_depgraph()
348
349             if 'world' in pkgs_to_build:
350                     pkgs_to_build.remove('world')
351                     for t in __world_target:
352                             pkgs_to_build.append(t)
353
354             oe.event.fire(oe.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
355
356             for k in pkgs_to_build:
357                     if buildPackage(k) == 0:
358                             oe.error("Build of " + k + " failed")
359                             if make.options.abort:
360                                     sys.exit(1)
361
362             oe.event.fire(oe.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
363
364             print "Build statistics:"
365             print "  Attempted builds: %d" % __stats["attempt"]
366             if __stats["fail"] != 0:
367                     print "  Failed builds: %d" % __stats["fail"]
368             if __stats["deps"] != 0:
369                     print "  Dependencies not satisfied: %d" % __stats["deps"]
370             if __stats["fail"] != 0 or __stats["deps"] != 0:
371                     sys.exit(1)
372             sys.exit(0)
373
374     except KeyboardInterrupt:
375             print "\nNOTE: KeyboardInterrupt - Build not completed."