cooker.py: Move some functionality to the module init function
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
77         """
78         Build one task of a package, optionally build following task depends
79         """
80         bb.event.fire(bb.event.PkgStarted(item, the_data))
81         try:
82             if not build_depends:
83                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
84             if not self.configuration.dry_run:
85                 bb.build.exec_task('do_%s' % task, the_data)
86             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
87             return True
88         except bb.build.FuncFailed:
89             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
90             bb.event.fire(bb.event.PkgFailed(item, the_data))
91             raise
92         except bb.build.EventException, e:
93             event = e.args[1]
94             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
95             bb.event.fire(bb.event.PkgFailed(item, the_data))
96             raise
97
98     def tryBuild( self, fn, build_depends):
99         """
100         Build a provider and its dependencies. 
101         build_depends is a list of previous build dependencies (not runtime)
102         If build_depends is empty, we're dealing with a runtime depends
103         """
104
105         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
106
107         item = self.status.pkg_fn[fn]
108
109         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
110             return True
111
112         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
113
114     def showVersions( self ):
115         pkg_pn = self.status.pkg_pn
116         preferred_versions = {}
117         latest_versions = {}
118
119         # Sort by priority
120         for pn in pkg_pn.keys():
121             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
122             preferred_versions[pn] = (pref_ver, pref_file)
123             latest_versions[pn] = (last_ver, last_file)
124
125         pkg_list = pkg_pn.keys()
126         pkg_list.sort()
127
128         for p in pkg_list:
129             pref = preferred_versions[p]
130             latest = latest_versions[p]
131
132             if pref != latest:
133                 prefstr = pref[0][0] + "-" + pref[0][1]
134             else:
135                 prefstr = ""
136
137             print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
138                                         prefstr)
139
140
141     def showEnvironment( self ):
142         """Show the outer or per-package environment"""
143         if self.configuration.buildfile:
144             self.cb = None
145             self.bb_cache = bb.cache.init(self)
146             try:
147                 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
148             except IOError, e:
149                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
150             except Exception, e:
151                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
152         # emit variables and shell functions
153         try:
154             data.update_data( self.configuration.data )
155             data.emit_env(sys.__stdout__, self.configuration.data, True)
156         except Exception, e:
157             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
158         # emit the metadata which isnt valid shell
159         data.expandKeys( self.configuration.data )      
160         for e in self.configuration.data.keys():
161             if data.getVarFlag( e, 'python', self.configuration.data ):
162                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
163
164     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
165         """
166         Generate a task dependency graph. 
167
168         pkgs_to_build A list of packages that needs to be built
169         ignore_deps   A list of names where processing of dependencies
170                       should be stopped. e.g. dependencies that get
171         """
172
173         for dep in ignore_deps:
174             self.status.ignored_dependencies.add(dep)
175
176         localdata = data.createCopy(self.configuration.data)
177         bb.data.update_data(localdata)
178         bb.data.expandKeys(localdata)
179         taskdata = bb.taskdata.TaskData(self.configuration.abort)
180
181         runlist = []
182         try:
183             for k in pkgs_to_build:
184                 taskdata.add_provider(localdata, self.status, k)
185                 runlist.append([k, "do_%s" % self.configuration.cmd])
186             taskdata.add_unresolved(localdata, self.status)
187         except bb.providers.NoProvider:
188             sys.exit(1)
189         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
190         rq.prepare_runqueue()
191
192         seen_fnids = []  
193         depends_file = file('depends.dot', 'w' )
194         tdepends_file = file('task-depends.dot', 'w' )
195         print >> depends_file, "digraph depends {"
196         print >> tdepends_file, "digraph depends {"
197         rq.prio_map.reverse()
198         for task1 in range(len(rq.runq_fnid)):
199             task = rq.prio_map[task1]
200             taskname = rq.runq_task[task]
201             fnid = rq.runq_fnid[task]
202             fn = taskdata.fn_index[fnid]
203             pn = self.status.pkg_fn[fn]
204             version  = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
205             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
206             for dep in rq.runq_depends[task]:
207                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
208                 deppn = self.status.pkg_fn[depfn]
209                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
210             if fnid not in seen_fnids:
211                 seen_fnids.append(fnid)
212                 packages = []
213                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
214                 for depend in self.status.deps[fn]:
215                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
216                 rdepends = self.status.rundeps[fn]
217                 for package in rdepends:
218                     for rdepend in rdepends[package]:
219                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
220                     packages.append(package)
221                 rrecs = self.status.runrecs[fn]
222                 for package in rrecs:
223                     for rdepend in rrecs[package]:
224                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
225                     if not package in packages:
226                         packages.append(package)
227                 for package in packages:
228                     if package != pn:
229                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
230                         for depend in self.status.deps[fn]:
231                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
232                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
233                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
234                 #for rdep in taskdata.rdepids[fnid]:
235                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
236                 #for dep in taskdata.depids[fnid]:
237                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
238         print >> depends_file,  "}"
239         print >> tdepends_file,  "}"
240         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
241         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
242
243     def buildDepgraph( self ):
244         all_depends = self.status.all_depends
245         pn_provides = self.status.pn_provides
246
247         localdata = data.createCopy(self.configuration.data)
248         bb.data.update_data(localdata)
249         bb.data.expandKeys(localdata)
250
251         def calc_bbfile_priority(filename):
252             for (regex, pri) in self.status.bbfile_config_priorities:
253                 if regex.match(filename):
254                     return pri
255             return 0
256
257         # Handle PREFERRED_PROVIDERS
258         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
259             (providee, provider) = p.split(':')
260             if providee in self.status.preferred and self.status.preferred[providee] != provider:
261                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
262             self.status.preferred[providee] = provider
263
264         # Calculate priorities for each file
265         for p in self.status.pkg_fn.keys():
266             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
267
268     def buildWorldTargetList(self):
269         """
270          Build package list for "bitbake world"
271         """
272         all_depends = self.status.all_depends
273         pn_provides = self.status.pn_provides
274         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
275         for f in self.status.possible_world:
276             terminal = True
277             pn = self.status.pkg_fn[f]
278
279             for p in pn_provides[pn]:
280                 if p.startswith('virtual/'):
281                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
282                     terminal = False
283                     break
284                 for pf in self.status.providers[p]:
285                     if self.status.pkg_fn[pf] != pn:
286                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
287                         terminal = False
288                         break
289             if terminal:
290                 self.status.world_target.add(pn)
291
292             # drop reference count now
293             self.status.possible_world = None
294             self.status.all_depends    = None
295
296     def myProgressCallback( self, x, y, f, from_cache ):
297         """Update any tty with the progress change"""
298         if os.isatty(sys.stdout.fileno()):
299             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
300             sys.stdout.flush()
301         else:
302             if x == 1:
303                 sys.stdout.write("Parsing .bb files, please wait...")
304                 sys.stdout.flush()
305             if x == y:
306                 sys.stdout.write("done.")
307                 sys.stdout.flush()
308
309     def interactiveMode( self ):
310         """Drop off into a shell"""
311         try:
312             from bb import shell
313         except ImportError, details:
314             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
315         else:
316             bb.data.update_data( self.configuration.data )
317             bb.data.expandKeys( self.configuration.data )
318             shell.start( self )
319             sys.exit( 0 )
320
321     def parseConfigurationFile( self, afile ):
322         try:
323             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
324
325             # Add the handlers we inherited by INHERIT
326             # we need to do this manually as it is not guranteed
327             # we will pick up these classes... as we only INHERIT
328             # on .inc and .bb files but not on .conf
329             data = bb.data.createCopy( self.configuration.data )
330             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
331             for inherit in inherits:
332                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
333
334             # FIXME: This assumes that we included at least one .inc file
335             for var in bb.data.keys(data):
336                 if bb.data.getVarFlag(var, 'handler', data):
337                     bb.event.register(var,bb.data.getVar(var, data))
338
339         except IOError:
340             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
341         except bb.parse.ParseError, details:
342             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
343
344     def handleCollections( self, collections ):
345         """Handle collections"""
346         if collections:
347             collection_list = collections.split()
348             for c in collection_list:
349                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
350                 if regex == None:
351                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
352                     continue
353                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
354                 if priority == None:
355                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
356                     continue
357                 try:
358                     cre = re.compile(regex)
359                 except re.error:
360                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
361                     continue
362                 try:
363                     pri = int(priority)
364                     self.status.bbfile_config_priorities.append((cre, pri))
365                 except ValueError:
366                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
367
368
369     def cook(self):
370         """
371         We are building stuff here. We do the building
372         from here. By default we try to execute task
373         build.
374         """
375
376         if self.configuration.show_environment:
377             self.showEnvironment()
378             sys.exit( 0 )
379
380         # inject custom variables
381         if not bb.data.getVar("BUILDNAME", self.configuration.data):
382             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
383         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
384
385         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
386
387         if self.configuration.interactive:
388             self.interactiveMode()
389
390         if self.configuration.buildfile is not None:
391             bf = os.path.abspath( self.configuration.buildfile )
392             try:
393                 os.stat(bf)
394             except OSError:
395                 (filelist, masked) = self.collect_bbfiles()
396                 regexp = re.compile(self.configuration.buildfile)
397                 matches = []
398                 for f in filelist:
399                     if regexp.search(f) and os.path.isfile(f):
400                         bf = f
401                         matches.append(f)
402                 if len(matches) != 1:
403                     bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
404                     for f in matches:
405                         bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
406                     sys.exit(1)
407                 bf = matches[0]             
408
409             bbfile_data = bb.parse.handle(bf, self.configuration.data)
410
411             # Remove stamp for target if force mode active
412             if self.configuration.force:
413                 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
414                 bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
415
416             item = bb.data.getVar('PN', bbfile_data, 1)
417             try:
418                 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
419             except bb.build.EventException:
420                 bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
421
422             sys.exit(0)
423
424         # initialise the parsing status now we know we will need deps
425         self.status = bb.cache.CacheData()
426
427         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
428         self.status.ignored_dependencies = Set( ignore.split() )
429
430         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
431
432         pkgs_to_build = self.configuration.pkgs_to_build
433
434         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
435         if bbpkgs:
436             pkgs_to_build.extend(bbpkgs.split())
437         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
438                              and not self.configuration.show_environment:
439                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
440                 print "for usage information."
441                 sys.exit(0)
442
443         # Import Psyco if available and not disabled
444         if not self.configuration.disable_psyco:
445             try:
446                 import psyco
447             except ImportError:
448                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
449             else:
450                 psyco.bind( self.parse_bbfiles )
451         else:
452             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
453
454         try:
455             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
456             (filelist, masked) = self.collect_bbfiles()
457             self.parse_bbfiles(filelist, masked, self.myProgressCallback)
458             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
459             print
460             if self.configuration.parse_only:
461                 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
462                 return
463
464
465             self.buildDepgraph()
466
467             if self.configuration.show_versions:
468                 self.showVersions()
469                 sys.exit( 0 )
470             if 'world' in pkgs_to_build:
471                 self.buildWorldTargetList()
472                 pkgs_to_build.remove('world')
473                 for t in self.status.world_target:
474                     pkgs_to_build.append(t)
475
476             if self.configuration.dot_graph:
477                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
478                 sys.exit( 0 )
479
480             bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
481
482             localdata = data.createCopy(self.configuration.data)
483             bb.data.update_data(localdata)
484             bb.data.expandKeys(localdata)
485
486             taskdata = bb.taskdata.TaskData(self.configuration.abort)
487
488             runlist = []
489             try:
490                 for k in pkgs_to_build:
491                     taskdata.add_provider(localdata, self.status, k)
492                     runlist.append([k, "do_%s" % self.configuration.cmd])
493                 taskdata.add_unresolved(localdata, self.status)
494             except bb.providers.NoProvider:
495                 sys.exit(1)
496
497             rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
498             rq.prepare_runqueue()
499             try:
500                 failures = rq.execute_runqueue()
501             except runqueue.TaskFailure, fnids:
502                 for fnid in fnids:
503                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
504                 sys.exit(1)
505             bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
506
507             sys.exit(0)
508
509         except KeyboardInterrupt:
510             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
511             sys.exit(1)
512
513     def get_bbfiles( self, path = os.getcwd() ):
514         """Get list of default .bb files by reading out the current directory"""
515         contents = os.listdir(path)
516         bbfiles = []
517         for f in contents:
518             (root, ext) = os.path.splitext(f)
519             if ext == ".bb":
520                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
521         return bbfiles
522
523     def find_bbfiles( self, path ):
524         """Find all the .bb files in a directory"""
525         from os.path import join
526
527         found = []
528         for dir, dirs, files in os.walk(path):
529             for ignored in ('SCCS', 'CVS', '.svn'):
530                 if ignored in dirs:
531                     dirs.remove(ignored)
532             found += [join(dir,f) for f in files if f.endswith('.bb')]
533
534         return found
535
536     def collect_bbfiles( self ):
537         """Collect all available .bb build files"""
538         parsed, cached, skipped, masked = 0, 0, 0, 0
539         self.bb_cache = bb.cache.init(self)
540
541         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
542         data.setVar("BBFILES", " ".join(files), self.configuration.data)
543
544         if not len(files):
545             files = self.get_bbfiles()
546
547         if not len(files):
548             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
549
550         newfiles = []
551         for f in files:
552             if os.path.isdir(f):
553                 dirfiles = self.find_bbfiles(f)
554                 if dirfiles:
555                     newfiles += dirfiles
556                     continue
557             newfiles += glob.glob(f) or [ f ]
558
559         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
560
561         if not bbmask:
562             return (newfiles, 0)
563
564         try:
565             bbmask_compiled = re.compile(bbmask)
566         except sre_constants.error:
567             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
568
569         finalfiles = []
570         for i in xrange( len( newfiles ) ):
571             f = newfiles[i]
572             if bbmask and bbmask_compiled.search(f):
573                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
574                 masked += 1
575                 continue
576             finalfiles.append(f)
577
578         return (finalfiles, masked)
579
580     def parse_bbfiles(self, filelist, masked, progressCallback = None):
581         parsed, cached, skipped = 0, 0, 0
582         for i in xrange( len( filelist ) ):
583             f = filelist[i]
584
585             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
586
587             # read a file's metadata
588             try:
589                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
590                 if skip:
591                     skipped += 1
592                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
593                     self.bb_cache.skip(f)
594                     continue
595                 elif fromCache: cached += 1
596                 else: parsed += 1
597                 deps = None
598
599                 # Disabled by RP as was no longer functional
600                 # allow metadata files to add items to BBFILES
601                 #data.update_data(self.pkgdata[f])
602                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
603                 #if addbbfiles:
604                 #    for aof in addbbfiles.split():
605                 #        if not files.count(aof):
606                 #            if not os.path.isabs(aof):
607                 #                aof = os.path.join(os.path.dirname(f),aof)
608                 #            files.append(aof)
609
610                 self.bb_cache.handle_data(f, self.status)
611
612                 # now inform the caller
613                 if progressCallback is not None:
614                     progressCallback( i + 1, len( filelist ), f, fromCache )
615
616             except IOError, e:
617                 self.bb_cache.remove(f)
618                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
619                 pass
620             except KeyboardInterrupt:
621                 self.bb_cache.sync()
622                 raise
623             except Exception, e:
624                 self.bb_cache.remove(f)
625                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
626             except:
627                 self.bb_cache.remove(f)
628                 raise
629
630         if progressCallback is not None:
631             print "\r" # need newline after Handling Bitbake files message
632             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
633
634         self.bb_cache.sync()