cooker.py: Warn about malformed PREFERRED_PROVIDERS (#1072) [backport]
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76         #
77         # TOSTOP must not be set or our children will hang when they output
78         #
79         fd = sys.stdout.fileno()
80         if os.isatty(fd):
81             import termios
82             tcattr = termios.tcgetattr(fd)
83             if tcattr[3] & termios.TOSTOP:
84                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
85                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
86                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
87
88         # Change nice level if we're asked to
89         nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
90         if nice:
91             curnice = os.nice(0)
92             nice = int(nice) - curnice
93             bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
94  
95
96     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
97         """
98         Build one task of a package, optionally build following task depends
99         """
100         bb.event.fire(bb.event.PkgStarted(item, the_data))
101         try:
102             if not build_depends:
103                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
104             if not self.configuration.dry_run:
105                 bb.build.exec_task('do_%s' % task, the_data)
106             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
107             return True
108         except bb.build.FuncFailed:
109             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
110             bb.event.fire(bb.event.PkgFailed(item, the_data))
111             raise
112         except bb.build.EventException, e:
113             event = e.args[1]
114             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
115             bb.event.fire(bb.event.PkgFailed(item, the_data))
116             raise
117
118     def tryBuild( self, fn, build_depends):
119         """
120         Build a provider and its dependencies. 
121         build_depends is a list of previous build dependencies (not runtime)
122         If build_depends is empty, we're dealing with a runtime depends
123         """
124
125         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
126
127         item = self.status.pkg_fn[fn]
128
129         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
130             return True
131
132         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
133
134     def showVersions(self):
135         pkg_pn = self.status.pkg_pn
136         preferred_versions = {}
137         latest_versions = {}
138
139         # Sort by priority
140         for pn in pkg_pn.keys():
141             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
142             preferred_versions[pn] = (pref_ver, pref_file)
143             latest_versions[pn] = (last_ver, last_file)
144
145         pkg_list = pkg_pn.keys()
146         pkg_list.sort()
147
148         for p in pkg_list:
149             pref = preferred_versions[p]
150             latest = latest_versions[p]
151
152             if pref != latest:
153                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
154             else:
155                 prefstr = ""
156
157             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
158                                         prefstr)
159
160
161     def showEnvironment( self ):
162         """Show the outer or per-package environment"""
163         if self.configuration.buildfile:
164             self.cb = None
165             self.bb_cache = bb.cache.init(self)
166             bf = self.matchFile(self.configuration.buildfile)
167             try:
168                 self.configuration.data = self.bb_cache.loadDataFull(bf, self.configuration.data)
169             except IOError, e:
170                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (bf, e))
171             except Exception, e:
172                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
173         # emit variables and shell functions
174         try:
175             data.update_data( self.configuration.data )
176             data.emit_env(sys.__stdout__, self.configuration.data, True)
177         except Exception, e:
178             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
179         # emit the metadata which isnt valid shell
180         data.expandKeys( self.configuration.data )      
181         for e in self.configuration.data.keys():
182             if data.getVarFlag( e, 'python', self.configuration.data ):
183                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
184
185     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
186         """
187         Generate a task dependency graph. 
188
189         pkgs_to_build A list of packages that needs to be built
190         ignore_deps   A list of names where processing of dependencies
191                       should be stopped. e.g. dependencies that get
192         """
193
194         for dep in ignore_deps:
195             self.status.ignored_dependencies.add(dep)
196
197         localdata = data.createCopy(self.configuration.data)
198         bb.data.update_data(localdata)
199         bb.data.expandKeys(localdata)
200         taskdata = bb.taskdata.TaskData(self.configuration.abort)
201
202         runlist = []
203         try:
204             for k in pkgs_to_build:
205                 taskdata.add_provider(localdata, self.status, k)
206                 runlist.append([k, "do_%s" % self.configuration.cmd])
207             taskdata.add_unresolved(localdata, self.status)
208         except bb.providers.NoProvider:
209             sys.exit(1)
210         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
211         rq.prepare_runqueue()
212
213         seen_fnids = []  
214         depends_file = file('depends.dot', 'w' )
215         tdepends_file = file('task-depends.dot', 'w' )
216         print >> depends_file, "digraph depends {"
217         print >> tdepends_file, "digraph depends {"
218         rq.prio_map.reverse()
219         for task1 in range(len(rq.runq_fnid)):
220             task = rq.prio_map[task1]
221             taskname = rq.runq_task[task]
222             fnid = rq.runq_fnid[task]
223             fn = taskdata.fn_index[fnid]
224             pn = self.status.pkg_fn[fn]
225             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
226             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
227             for dep in rq.runq_depends[task]:
228                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
229                 deppn = self.status.pkg_fn[depfn]
230                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
231             if fnid not in seen_fnids:
232                 seen_fnids.append(fnid)
233                 packages = []
234                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
235                 for depend in self.status.deps[fn]:
236                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
237                 rdepends = self.status.rundeps[fn]
238                 for package in rdepends:
239                     for rdepend in rdepends[package]:
240                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
241                     packages.append(package)
242                 rrecs = self.status.runrecs[fn]
243                 for package in rrecs:
244                     for rdepend in rrecs[package]:
245                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
246                     if not package in packages:
247                         packages.append(package)
248                 for package in packages:
249                     if package != pn:
250                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
251                         for depend in self.status.deps[fn]:
252                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
253                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
254                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
255                 #for rdep in taskdata.rdepids[fnid]:
256                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
257                 #for dep in taskdata.depids[fnid]:
258                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
259         print >> depends_file,  "}"
260         print >> tdepends_file,  "}"
261         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
262         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
263
264     def buildDepgraph( self ):
265         all_depends = self.status.all_depends
266         pn_provides = self.status.pn_provides
267
268         localdata = data.createCopy(self.configuration.data)
269         bb.data.update_data(localdata)
270         bb.data.expandKeys(localdata)
271
272         def calc_bbfile_priority(filename):
273             for (regex, pri) in self.status.bbfile_config_priorities:
274                 if regex.match(filename):
275                     return pri
276             return 0
277
278         # Handle PREFERRED_PROVIDERS
279         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
280             try:
281                 (providee, provider) = p.split(':')
282             except:
283                 bb.msg.error(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
284                 continue
285             if providee in self.status.preferred and self.status.preferred[providee] != provider:
286                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
287             self.status.preferred[providee] = provider
288
289         # Calculate priorities for each file
290         for p in self.status.pkg_fn.keys():
291             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
292
293     def buildWorldTargetList(self):
294         """
295          Build package list for "bitbake world"
296         """
297         all_depends = self.status.all_depends
298         pn_provides = self.status.pn_provides
299         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
300         for f in self.status.possible_world:
301             terminal = True
302             pn = self.status.pkg_fn[f]
303
304             for p in pn_provides[pn]:
305                 if p.startswith('virtual/'):
306                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
307                     terminal = False
308                     break
309                 for pf in self.status.providers[p]:
310                     if self.status.pkg_fn[pf] != pn:
311                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
312                         terminal = False
313                         break
314             if terminal:
315                 self.status.world_target.add(pn)
316
317             # drop reference count now
318             self.status.possible_world = None
319             self.status.all_depends    = None
320
321     def myProgressCallback( self, x, y, f, from_cache ):
322         """Update any tty with the progress change"""
323         if os.isatty(sys.stdout.fileno()):
324             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
325             sys.stdout.flush()
326         else:
327             if x == 1:
328                 sys.stdout.write("Parsing .bb files, please wait...")
329                 sys.stdout.flush()
330             if x == y:
331                 sys.stdout.write("done.")
332                 sys.stdout.flush()
333
334     def interactiveMode( self ):
335         """Drop off into a shell"""
336         try:
337             from bb import shell
338         except ImportError, details:
339             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
340         else:
341             bb.data.update_data( self.configuration.data )
342             bb.data.expandKeys( self.configuration.data )
343             shell.start( self )
344             sys.exit( 0 )
345
346     def parseConfigurationFile( self, afile ):
347         try:
348             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
349
350             # Add the handlers we inherited by INHERIT
351             # we need to do this manually as it is not guranteed
352             # we will pick up these classes... as we only INHERIT
353             # on .inc and .bb files but not on .conf
354             data = bb.data.createCopy( self.configuration.data )
355             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
356             for inherit in inherits:
357                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
358
359             # FIXME: This assumes that we included at least one .inc file
360             for var in bb.data.keys(data):
361                 if bb.data.getVarFlag(var, 'handler', data):
362                     bb.event.register(var,bb.data.getVar(var, data))
363
364             bb.fetch.fetcher_init(self.configuration.data)
365
366             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
367
368         except IOError:
369             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
370         except bb.parse.ParseError, details:
371             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
372
373     def handleCollections( self, collections ):
374         """Handle collections"""
375         if collections:
376             collection_list = collections.split()
377             for c in collection_list:
378                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
379                 if regex == None:
380                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
381                     continue
382                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
383                 if priority == None:
384                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
385                     continue
386                 try:
387                     cre = re.compile(regex)
388                 except re.error:
389                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
390                     continue
391                 try:
392                     pri = int(priority)
393                     self.status.bbfile_config_priorities.append((cre, pri))
394                 except ValueError:
395                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
396
397     def buildSetVars(self):
398         """
399         Setup any variables needed before starting a build
400         """
401         if not bb.data.getVar("BUILDNAME", self.configuration.data):
402             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
403         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
404
405     def matchFile(self, buildfile):
406         """
407         Convert the fragment buildfile into a real file
408         Error if there are too many matches
409         """
410         bf = os.path.abspath(buildfile)
411         try:
412             os.stat(bf)
413             return bf
414         except OSError:
415             (filelist, masked) = self.collect_bbfiles()
416             regexp = re.compile(buildfile)
417             matches = []
418             for f in filelist:
419                 if regexp.search(f) and os.path.isfile(f):
420                     bf = f
421                     matches.append(f)
422             if len(matches) != 1:
423                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
424                 for f in matches:
425                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
426                 sys.exit(1)
427             return matches[0]               
428
429     def buildFile(self, buildfile):
430         """
431         Build the file matching regexp buildfile
432         """
433
434         bf = self.matchFile(buildfile)
435
436         bbfile_data = bb.parse.handle(bf, self.configuration.data)
437
438         # Remove stamp for target if force mode active
439         if self.configuration.force:
440             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
441             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
442
443         item = bb.data.getVar('PN', bbfile_data, 1)
444         try:
445             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
446         except bb.build.EventException:
447             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
448
449         sys.exit(0)
450
451     def buildTargets(self, targets):
452         """
453         Attempt to build the targets specified
454         """
455
456         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
457         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
458
459         localdata = data.createCopy(self.configuration.data)
460         bb.data.update_data(localdata)
461         bb.data.expandKeys(localdata)
462
463         taskdata = bb.taskdata.TaskData(self.configuration.abort)
464
465         runlist = []
466         try:
467             for k in targets:
468                 taskdata.add_provider(localdata, self.status, k)
469                 runlist.append([k, "do_%s" % self.configuration.cmd])
470             taskdata.add_unresolved(localdata, self.status)
471         except bb.providers.NoProvider:
472             sys.exit(1)
473
474         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
475         rq.prepare_runqueue()
476         try:
477             failures = rq.execute_runqueue()
478         except runqueue.TaskFailure, fnids:
479             for fnid in fnids:
480                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
481             sys.exit(1)
482         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
483
484         sys.exit(0)
485
486     def updateCache(self):
487         # Import Psyco if available and not disabled
488         import platform
489         if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
490             if not self.configuration.disable_psyco:
491                 try:
492                     import psyco
493                 except ImportError:
494                     bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
495                 else:
496                     psyco.bind( self.parse_bbfiles )
497             else:
498                 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
499
500         self.status = bb.cache.CacheData()
501
502         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
503         self.status.ignored_dependencies = Set( ignore.split() )
504
505         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
506
507         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
508         (filelist, masked) = self.collect_bbfiles()
509         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
510         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
511
512         self.buildDepgraph()
513
514     def cook(self):
515         """
516         We are building stuff here. We do the building
517         from here. By default we try to execute task
518         build.
519         """
520
521         if self.configuration.show_environment:
522             self.showEnvironment()
523             sys.exit( 0 )
524
525         self.buildSetVars()
526
527         if self.configuration.interactive:
528             self.interactiveMode()
529
530         if self.configuration.buildfile is not None:
531             return self.buildFile(self.configuration.buildfile)
532
533         # initialise the parsing status now we know we will need deps
534         self.updateCache()
535
536         if self.configuration.parse_only:
537             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
538             return 0
539
540         pkgs_to_build = self.configuration.pkgs_to_build
541
542         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
543         if bbpkgs:
544             pkgs_to_build.extend(bbpkgs.split())
545         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
546                              and not self.configuration.show_environment:
547                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
548                 print "for usage information."
549                 sys.exit(0)
550
551         try:
552             if self.configuration.show_versions:
553                 self.showVersions()
554                 sys.exit( 0 )
555             if 'world' in pkgs_to_build:
556                 self.buildWorldTargetList()
557                 pkgs_to_build.remove('world')
558                 for t in self.status.world_target:
559                     pkgs_to_build.append(t)
560
561             if self.configuration.dot_graph:
562                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
563                 sys.exit( 0 )
564
565             return self.buildTargets(pkgs_to_build)
566
567         except KeyboardInterrupt:
568             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
569             sys.exit(1)
570
571     def get_bbfiles( self, path = os.getcwd() ):
572         """Get list of default .bb files by reading out the current directory"""
573         contents = os.listdir(path)
574         bbfiles = []
575         for f in contents:
576             (root, ext) = os.path.splitext(f)
577             if ext == ".bb":
578                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
579         return bbfiles
580
581     def find_bbfiles( self, path ):
582         """Find all the .bb files in a directory"""
583         from os.path import join
584
585         found = []
586         for dir, dirs, files in os.walk(path):
587             for ignored in ('SCCS', 'CVS', '.svn'):
588                 if ignored in dirs:
589                     dirs.remove(ignored)
590             found += [join(dir,f) for f in files if f.endswith('.bb')]
591
592         return found
593
594     def collect_bbfiles( self ):
595         """Collect all available .bb build files"""
596         parsed, cached, skipped, masked = 0, 0, 0, 0
597         self.bb_cache = bb.cache.init(self)
598
599         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
600         data.setVar("BBFILES", " ".join(files), self.configuration.data)
601
602         if not len(files):
603             files = self.get_bbfiles()
604
605         if not len(files):
606             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
607
608         newfiles = []
609         for f in files:
610             if os.path.isdir(f):
611                 dirfiles = self.find_bbfiles(f)
612                 if dirfiles:
613                     newfiles += dirfiles
614                     continue
615             newfiles += glob.glob(f) or [ f ]
616
617         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
618
619         if not bbmask:
620             return (newfiles, 0)
621
622         try:
623             bbmask_compiled = re.compile(bbmask)
624         except sre_constants.error:
625             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
626
627         finalfiles = []
628         for i in xrange( len( newfiles ) ):
629             f = newfiles[i]
630             if bbmask and bbmask_compiled.search(f):
631                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
632                 masked += 1
633                 continue
634             finalfiles.append(f)
635
636         return (finalfiles, masked)
637
638     def parse_bbfiles(self, filelist, masked, progressCallback = None):
639         parsed, cached, skipped, error = 0, 0, 0, 0
640         for i in xrange( len( filelist ) ):
641             f = filelist[i]
642
643             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
644
645             # read a file's metadata
646             try:
647                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
648                 if skip:
649                     skipped += 1
650                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
651                     self.bb_cache.skip(f)
652                     continue
653                 elif fromCache: cached += 1
654                 else: parsed += 1
655                 deps = None
656
657                 # Disabled by RP as was no longer functional
658                 # allow metadata files to add items to BBFILES
659                 #data.update_data(self.pkgdata[f])
660                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
661                 #if addbbfiles:
662                 #    for aof in addbbfiles.split():
663                 #        if not files.count(aof):
664                 #            if not os.path.isabs(aof):
665                 #                aof = os.path.join(os.path.dirname(f),aof)
666                 #            files.append(aof)
667
668                 self.bb_cache.handle_data(f, self.status)
669
670                 # now inform the caller
671                 if progressCallback is not None:
672                     progressCallback( i + 1, len( filelist ), f, fromCache )
673
674             except IOError, e:
675                 self.bb_cache.remove(f)
676                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
677                 pass
678             except KeyboardInterrupt:
679                 self.bb_cache.sync()
680                 raise
681             except Exception, e:
682                 error += 1
683                 self.bb_cache.remove(f)
684                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
685             except:
686                 self.bb_cache.remove(f)
687                 raise
688
689         if progressCallback is not None:
690             print "\r" # need newline after Handling Bitbake files message
691             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
692
693         self.bb_cache.sync()
694
695         if error > 0:
696             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")