cooker.py, runqueue.py: Move statistics implementation from cooker to runqueue
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     Statistics = BBStatistics           # make it visible from the shell
42
43     def __init__( self ):
44         self.status = None
45
46         self.cache = None
47         self.bb_cache = None
48
49     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
50         """
51         Build one task of a package, optionally build following task depends
52         """
53         bb.event.fire(bb.event.PkgStarted(item, the_data))
54         try:
55             if not build_depends:
56                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
57             if not self.configuration.dry_run:
58                 bb.build.exec_task('do_%s' % task, the_data)
59             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
60             return True
61         except bb.build.FuncFailed:
62             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
63             bb.event.fire(bb.event.PkgFailed(item, the_data))
64             raise
65         except bb.build.EventException, e:
66             event = e.args[1]
67             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
68             bb.event.fire(bb.event.PkgFailed(item, the_data))
69             raise
70
71     def tryBuild( self, fn, build_depends):
72         """
73         Build a provider and its dependencies. 
74         build_depends is a list of previous build dependencies (not runtime)
75         If build_depends is empty, we're dealing with a runtime depends
76         """
77
78         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
79
80         item = self.status.pkg_fn[fn]
81
82         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
83             return True
84
85         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
86
87     def showVersions( self ):
88         pkg_pn = self.status.pkg_pn
89         preferred_versions = {}
90         latest_versions = {}
91
92         # Sort by priority
93         for pn in pkg_pn.keys():
94             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
95             preferred_versions[pn] = (pref_ver, pref_file)
96             latest_versions[pn] = (last_ver, last_file)
97
98         pkg_list = pkg_pn.keys()
99         pkg_list.sort()
100
101         for p in pkg_list:
102             pref = preferred_versions[p]
103             latest = latest_versions[p]
104
105             if pref != latest:
106                 prefstr = pref[0][0] + "-" + pref[0][1]
107             else:
108                 prefstr = ""
109
110             print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
111                                         prefstr)
112
113
114     def showEnvironment( self ):
115         """Show the outer or per-package environment"""
116         if self.configuration.buildfile:
117             self.cb = None
118             self.bb_cache = bb.cache.init(self)
119             try:
120                 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
121             except IOError, e:
122                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
123             except Exception, e:
124                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
125         # emit variables and shell functions
126         try:
127             data.update_data( self.configuration.data )
128             data.emit_env(sys.__stdout__, self.configuration.data, True)
129         except Exception, e:
130             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
131         # emit the metadata which isnt valid shell
132         data.expandKeys( self.configuration.data )      
133         for e in self.configuration.data.keys():
134             if data.getVarFlag( e, 'python', self.configuration.data ):
135                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
136
137     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
138         """
139         Generate a task dependency graph. 
140
141         pkgs_to_build A list of packages that needs to be built
142         ignore_deps   A list of names where processing of dependencies
143                       should be stopped. e.g. dependencies that get
144         """
145
146         for dep in ignore_deps:
147             self.status.ignored_dependencies.add(dep)
148
149         localdata = data.createCopy(self.configuration.data)
150         bb.data.update_data(localdata)
151         bb.data.expandKeys(localdata)
152         taskdata = bb.taskdata.TaskData(self.configuration.abort)
153
154         runlist = []
155         try:
156             for k in pkgs_to_build:
157                 taskdata.add_provider(localdata, self.status, k)
158                 runlist.append([k, "do_%s" % self.configuration.cmd])
159             taskdata.add_unresolved(localdata, self.status)
160         except bb.providers.NoProvider:
161             sys.exit(1)
162         rq = bb.runqueue.RunQueue()
163         rq.prepare_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
164
165         seen_fnids = []  
166         depends_file = file('depends.dot', 'w' )
167         tdepends_file = file('task-depends.dot', 'w' )
168         print >> depends_file, "digraph depends {"
169         print >> tdepends_file, "digraph depends {"
170         rq.prio_map.reverse()
171         for task1 in range(len(rq.runq_fnid)):
172             task = rq.prio_map[task1]
173             taskname = rq.runq_task[task]
174             fnid = rq.runq_fnid[task]
175             fn = taskdata.fn_index[fnid]
176             pn = self.status.pkg_fn[fn]
177             version  = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
178             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
179             for dep in rq.runq_depends[task]:
180                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
181                 deppn = self.status.pkg_fn[depfn]
182                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
183             if fnid not in seen_fnids:
184                 seen_fnids.append(fnid)
185                 packages = []
186                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
187                 for depend in self.status.deps[fn]:
188                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
189                 rdepends = self.status.rundeps[fn]
190                 for package in rdepends:
191                     for rdepend in rdepends[package]:
192                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
193                     packages.append(package)
194                 rrecs = self.status.runrecs[fn]
195                 for package in rrecs:
196                     for rdepend in rrecs[package]:
197                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
198                     if not package in packages:
199                         packages.append(package)
200                 for package in packages:
201                     if package != pn:
202                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
203                         for depend in self.status.deps[fn]:
204                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
205                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
206                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
207                 #for rdep in taskdata.rdepids[fnid]:
208                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
209                 #for dep in taskdata.depids[fnid]:
210                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
211         print >> depends_file,  "}"
212         print >> tdepends_file,  "}"
213         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
214         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
215
216     def buildDepgraph( self ):
217         all_depends = self.status.all_depends
218         pn_provides = self.status.pn_provides
219
220         localdata = data.createCopy(self.configuration.data)
221         bb.data.update_data(localdata)
222         bb.data.expandKeys(localdata)
223
224         def calc_bbfile_priority(filename):
225             for (regex, pri) in self.status.bbfile_config_priorities:
226                 if regex.match(filename):
227                     return pri
228             return 0
229
230         # Handle PREFERRED_PROVIDERS
231         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
232             (providee, provider) = p.split(':')
233             if providee in self.status.preferred and self.status.preferred[providee] != provider:
234                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
235             self.status.preferred[providee] = provider
236
237         # Calculate priorities for each file
238         for p in self.status.pkg_fn.keys():
239             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
240
241     def buildWorldTargetList(self):
242         """
243          Build package list for "bitbake world"
244         """
245         all_depends = self.status.all_depends
246         pn_provides = self.status.pn_provides
247         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
248         for f in self.status.possible_world:
249             terminal = True
250             pn = self.status.pkg_fn[f]
251
252             for p in pn_provides[pn]:
253                 if p.startswith('virtual/'):
254                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
255                     terminal = False
256                     break
257                 for pf in self.status.providers[p]:
258                     if self.status.pkg_fn[pf] != pn:
259                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
260                         terminal = False
261                         break
262             if terminal:
263                 self.status.world_target.add(pn)
264
265             # drop reference count now
266             self.status.possible_world = None
267             self.status.all_depends    = None
268
269     def myProgressCallback( self, x, y, f, from_cache ):
270         """Update any tty with the progress change"""
271         if os.isatty(sys.stdout.fileno()):
272             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
273             sys.stdout.flush()
274         else:
275             if x == 1:
276                 sys.stdout.write("Parsing .bb files, please wait...")
277                 sys.stdout.flush()
278             if x == y:
279                 sys.stdout.write("done.")
280                 sys.stdout.flush()
281
282     def interactiveMode( self ):
283         """Drop off into a shell"""
284         try:
285             from bb import shell
286         except ImportError, details:
287             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
288         else:
289             bb.data.update_data( self.configuration.data )
290             bb.data.expandKeys( self.configuration.data )
291             shell.start( self )
292             sys.exit( 0 )
293
294     def parseConfigurationFile( self, afile ):
295         try:
296             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
297
298             # Add the handlers we inherited by INHERIT
299             # we need to do this manually as it is not guranteed
300             # we will pick up these classes... as we only INHERIT
301             # on .inc and .bb files but not on .conf
302             data = bb.data.createCopy( self.configuration.data )
303             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
304             for inherit in inherits:
305                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
306
307             # FIXME: This assumes that we included at least one .inc file
308             for var in bb.data.keys(data):
309                 if bb.data.getVarFlag(var, 'handler', data):
310                     bb.event.register(var,bb.data.getVar(var, data))
311
312         except IOError:
313             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
314         except bb.parse.ParseError, details:
315             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
316
317     def handleCollections( self, collections ):
318         """Handle collections"""
319         if collections:
320             collection_list = collections.split()
321             for c in collection_list:
322                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
323                 if regex == None:
324                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
325                     continue
326                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
327                 if priority == None:
328                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
329                     continue
330                 try:
331                     cre = re.compile(regex)
332                 except re.error:
333                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
334                     continue
335                 try:
336                     pri = int(priority)
337                     self.status.bbfile_config_priorities.append((cre, pri))
338                 except ValueError:
339                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
340
341
342     def cook(self, configuration):
343         """
344         We are building stuff here. We do the building
345         from here. By default we try to execute task
346         build.
347         """
348
349         self.configuration = configuration
350
351         if self.configuration.verbose:
352             bb.msg.set_verbose(True)
353
354         if self.configuration.debug:
355             bb.msg.set_debug_level(self.configuration.debug)
356         else:
357             bb.msg.set_debug_level(0)
358
359         if self.configuration.debug_domains:
360             bb.msg.set_debug_domains(self.configuration.debug_domains)
361
362         self.configuration.data = bb.data.init()
363
364         for f in self.configuration.file:
365             self.parseConfigurationFile( f )
366
367         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
368
369         if not self.configuration.cmd:
370             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
371
372         #
373         # Special updated configuration we use for firing events
374         #
375         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
376         bb.data.update_data(self.configuration.event_data)
377
378         if self.configuration.show_environment:
379             self.showEnvironment()
380             sys.exit( 0 )
381
382         # inject custom variables
383         if not bb.data.getVar("BUILDNAME", self.configuration.data):
384             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
385         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
386
387         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
388
389         if self.configuration.interactive:
390             self.interactiveMode()
391
392         if self.configuration.buildfile is not None:
393             bf = os.path.abspath( self.configuration.buildfile )
394             try:
395                 os.stat(bf)
396             except OSError:
397                 (filelist, masked) = self.collect_bbfiles()
398                 regexp = re.compile(self.configuration.buildfile)
399                 matches = []
400                 for f in filelist:
401                     if regexp.search(f) and os.path.isfile(f):
402                         bf = f
403                         matches.append(f)
404                 if len(matches) != 1:
405                     bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
406                     for f in matches:
407                         bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
408                     sys.exit(1)
409                 bf = matches[0]             
410
411             bbfile_data = bb.parse.handle(bf, self.configuration.data)
412
413             # Remove stamp for target if force mode active
414             if self.configuration.force:
415                 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
416                 bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
417
418             item = bb.data.getVar('PN', bbfile_data, 1)
419             try:
420                 self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
421             except bb.build.EventException:
422                 bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
423
424             sys.exit(0)
425
426         # initialise the parsing status now we know we will need deps
427         self.status = bb.cache.CacheData()
428
429         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
430         self.status.ignored_dependencies = Set( ignore.split() )
431
432         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
433
434         pkgs_to_build = self.configuration.pkgs_to_build
435
436         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
437         if bbpkgs:
438             pkgs_to_build.extend(bbpkgs.split())
439         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
440                              and not self.configuration.show_environment:
441                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
442                 print "for usage information."
443                 sys.exit(0)
444
445         # Import Psyco if available and not disabled
446         if not self.configuration.disable_psyco:
447             try:
448                 import psyco
449             except ImportError:
450                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
451             else:
452                 psyco.bind( self.parse_bbfiles )
453         else:
454             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
455
456         try:
457             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
458             (filelist, masked) = self.collect_bbfiles()
459             self.parse_bbfiles(filelist, masked, self.myProgressCallback)
460             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
461             print
462             if self.configuration.parse_only:
463                 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
464                 return
465
466
467             self.buildDepgraph()
468
469             if self.configuration.show_versions:
470                 self.showVersions()
471                 sys.exit( 0 )
472             if 'world' in pkgs_to_build:
473                 self.buildWorldTargetList()
474                 pkgs_to_build.remove('world')
475                 for t in self.status.world_target:
476                     pkgs_to_build.append(t)
477
478             if self.configuration.dot_graph:
479                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
480                 sys.exit( 0 )
481
482             bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
483
484             localdata = data.createCopy(self.configuration.data)
485             bb.data.update_data(localdata)
486             bb.data.expandKeys(localdata)
487
488             taskdata = bb.taskdata.TaskData(self.configuration.abort)
489
490             runlist = []
491             try:
492                 for k in pkgs_to_build:
493                     taskdata.add_provider(localdata, self.status, k)
494                     runlist.append([k, "do_%s" % self.configuration.cmd])
495                 taskdata.add_unresolved(localdata, self.status)
496             except bb.providers.NoProvider:
497                 sys.exit(1)
498
499             rq = bb.runqueue.RunQueue()
500             rq.prepare_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
501             try:
502                 failures = rq.execute_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
503             except runqueue.TaskFailure, fnids:
504                 for fnid in fnids:
505                     bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
506                 sys.exit(1)
507             bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
508
509             sys.exit(0)
510
511         except KeyboardInterrupt:
512             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
513             sys.exit(1)
514
515     def get_bbfiles( self, path = os.getcwd() ):
516         """Get list of default .bb files by reading out the current directory"""
517         contents = os.listdir(path)
518         bbfiles = []
519         for f in contents:
520             (root, ext) = os.path.splitext(f)
521             if ext == ".bb":
522                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
523         return bbfiles
524
525     def find_bbfiles( self, path ):
526         """Find all the .bb files in a directory"""
527         from os.path import join
528
529         found = []
530         for dir, dirs, files in os.walk(path):
531             for ignored in ('SCCS', 'CVS', '.svn'):
532                 if ignored in dirs:
533                     dirs.remove(ignored)
534             found += [join(dir,f) for f in files if f.endswith('.bb')]
535
536         return found
537
538     def collect_bbfiles( self ):
539         """Collect all available .bb build files"""
540         parsed, cached, skipped, masked = 0, 0, 0, 0
541         self.bb_cache = bb.cache.init(self)
542
543         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
544         data.setVar("BBFILES", " ".join(files), self.configuration.data)
545
546         if not len(files):
547             files = self.get_bbfiles()
548
549         if not len(files):
550             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
551
552         newfiles = []
553         for f in files:
554             if os.path.isdir(f):
555                 dirfiles = self.find_bbfiles(f)
556                 if dirfiles:
557                     newfiles += dirfiles
558                     continue
559             newfiles += glob.glob(f) or [ f ]
560
561         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
562
563         if not bbmask:
564             return (newfiles, 0)
565
566         try:
567             bbmask_compiled = re.compile(bbmask)
568         except sre_constants.error:
569             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
570
571         finalfiles = []
572         for i in xrange( len( newfiles ) ):
573             f = newfiles[i]
574             if bbmask and bbmask_compiled.search(f):
575                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
576                 masked += 1
577                 continue
578             finalfiles.append(f)
579
580         return (finalfiles, masked)
581
582     def parse_bbfiles(self, filelist, masked, progressCallback = None):
583         parsed, cached, skipped = 0, 0, 0
584         for i in xrange( len( filelist ) ):
585             f = filelist[i]
586
587             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
588
589             # read a file's metadata
590             try:
591                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
592                 if skip:
593                     skipped += 1
594                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
595                     self.bb_cache.skip(f)
596                     continue
597                 elif fromCache: cached += 1
598                 else: parsed += 1
599                 deps = None
600
601                 # Disabled by RP as was no longer functional
602                 # allow metadata files to add items to BBFILES
603                 #data.update_data(self.pkgdata[f])
604                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
605                 #if addbbfiles:
606                 #    for aof in addbbfiles.split():
607                 #        if not files.count(aof):
608                 #            if not os.path.isabs(aof):
609                 #                aof = os.path.join(os.path.dirname(f),aof)
610                 #            files.append(aof)
611
612                 self.bb_cache.handle_data(f, self.status)
613
614                 # now inform the caller
615                 if progressCallback is not None:
616                     progressCallback( i + 1, len( filelist ), f, fromCache )
617
618             except IOError, e:
619                 self.bb_cache.remove(f)
620                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
621                 pass
622             except KeyboardInterrupt:
623                 self.bb_cache.sync()
624                 raise
625             except Exception, e:
626                 self.bb_cache.remove(f)
627                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
628             except:
629                 self.bb_cache.remove(f)
630                 raise
631
632         if progressCallback is not None:
633             print "\r" # need newline after Handling Bitbake files message
634             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
635
636         self.bb_cache.sync()