cooker.py: Only try to change terminal flags on terminals
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76         #
77         # TOSTOP must not be set or our children will hang when they output
78         #
79         fd = sys.stdout.fileno()
80         if os.isatty(fd):
81             import termios
82             tcattr = termios.tcgetattr(fd)
83             if tcattr[3] & termios.TOSTOP:
84                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
85                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
86                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
87
88
89     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
90         """
91         Build one task of a package, optionally build following task depends
92         """
93         bb.event.fire(bb.event.PkgStarted(item, the_data))
94         try:
95             if not build_depends:
96                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
97             if not self.configuration.dry_run:
98                 bb.build.exec_task('do_%s' % task, the_data)
99             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
100             return True
101         except bb.build.FuncFailed:
102             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
103             bb.event.fire(bb.event.PkgFailed(item, the_data))
104             raise
105         except bb.build.EventException, e:
106             event = e.args[1]
107             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
108             bb.event.fire(bb.event.PkgFailed(item, the_data))
109             raise
110
111     def tryBuild( self, fn, build_depends):
112         """
113         Build a provider and its dependencies. 
114         build_depends is a list of previous build dependencies (not runtime)
115         If build_depends is empty, we're dealing with a runtime depends
116         """
117
118         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
119
120         item = self.status.pkg_fn[fn]
121
122         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
123             return True
124
125         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
126
127     def showVersions(self):
128         pkg_pn = self.status.pkg_pn
129         preferred_versions = {}
130         latest_versions = {}
131
132         # Sort by priority
133         for pn in pkg_pn.keys():
134             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
135             preferred_versions[pn] = (pref_ver, pref_file)
136             latest_versions[pn] = (last_ver, last_file)
137
138         pkg_list = pkg_pn.keys()
139         pkg_list.sort()
140
141         for p in pkg_list:
142             pref = preferred_versions[p]
143             latest = latest_versions[p]
144
145             if pref != latest:
146                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
147             else:
148                 prefstr = ""
149
150             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
151                                         prefstr)
152
153
154     def showEnvironment( self ):
155         """Show the outer or per-package environment"""
156         if self.configuration.buildfile:
157             self.cb = None
158             self.bb_cache = bb.cache.init(self)
159             bf = self.matchFile(self.configuration.buildfile)
160             try:
161                 self.configuration.data = self.bb_cache.loadDataFull(bf, self.configuration.data)
162             except IOError, e:
163                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (bf, e))
164             except Exception, e:
165                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
166         # emit variables and shell functions
167         try:
168             data.update_data( self.configuration.data )
169             data.emit_env(sys.__stdout__, self.configuration.data, True)
170         except Exception, e:
171             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
172         # emit the metadata which isnt valid shell
173         data.expandKeys( self.configuration.data )      
174         for e in self.configuration.data.keys():
175             if data.getVarFlag( e, 'python', self.configuration.data ):
176                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
177
178     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
179         """
180         Generate a task dependency graph. 
181
182         pkgs_to_build A list of packages that needs to be built
183         ignore_deps   A list of names where processing of dependencies
184                       should be stopped. e.g. dependencies that get
185         """
186
187         for dep in ignore_deps:
188             self.status.ignored_dependencies.add(dep)
189
190         localdata = data.createCopy(self.configuration.data)
191         bb.data.update_data(localdata)
192         bb.data.expandKeys(localdata)
193         taskdata = bb.taskdata.TaskData(self.configuration.abort)
194
195         runlist = []
196         try:
197             for k in pkgs_to_build:
198                 taskdata.add_provider(localdata, self.status, k)
199                 runlist.append([k, "do_%s" % self.configuration.cmd])
200             taskdata.add_unresolved(localdata, self.status)
201         except bb.providers.NoProvider:
202             sys.exit(1)
203         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
204         rq.prepare_runqueue()
205
206         seen_fnids = []  
207         depends_file = file('depends.dot', 'w' )
208         tdepends_file = file('task-depends.dot', 'w' )
209         print >> depends_file, "digraph depends {"
210         print >> tdepends_file, "digraph depends {"
211         rq.prio_map.reverse()
212         for task1 in range(len(rq.runq_fnid)):
213             task = rq.prio_map[task1]
214             taskname = rq.runq_task[task]
215             fnid = rq.runq_fnid[task]
216             fn = taskdata.fn_index[fnid]
217             pn = self.status.pkg_fn[fn]
218             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
219             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
220             for dep in rq.runq_depends[task]:
221                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
222                 deppn = self.status.pkg_fn[depfn]
223                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
224             if fnid not in seen_fnids:
225                 seen_fnids.append(fnid)
226                 packages = []
227                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
228                 for depend in self.status.deps[fn]:
229                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
230                 rdepends = self.status.rundeps[fn]
231                 for package in rdepends:
232                     for rdepend in rdepends[package]:
233                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
234                     packages.append(package)
235                 rrecs = self.status.runrecs[fn]
236                 for package in rrecs:
237                     for rdepend in rrecs[package]:
238                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
239                     if not package in packages:
240                         packages.append(package)
241                 for package in packages:
242                     if package != pn:
243                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
244                         for depend in self.status.deps[fn]:
245                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
246                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
247                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
248                 #for rdep in taskdata.rdepids[fnid]:
249                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
250                 #for dep in taskdata.depids[fnid]:
251                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
252         print >> depends_file,  "}"
253         print >> tdepends_file,  "}"
254         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
255         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
256
257     def buildDepgraph( self ):
258         all_depends = self.status.all_depends
259         pn_provides = self.status.pn_provides
260
261         localdata = data.createCopy(self.configuration.data)
262         bb.data.update_data(localdata)
263         bb.data.expandKeys(localdata)
264
265         def calc_bbfile_priority(filename):
266             for (regex, pri) in self.status.bbfile_config_priorities:
267                 if regex.match(filename):
268                     return pri
269             return 0
270
271         # Handle PREFERRED_PROVIDERS
272         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
273             (providee, provider) = p.split(':')
274             if providee in self.status.preferred and self.status.preferred[providee] != provider:
275                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
276             self.status.preferred[providee] = provider
277
278         # Calculate priorities for each file
279         for p in self.status.pkg_fn.keys():
280             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
281
282     def buildWorldTargetList(self):
283         """
284          Build package list for "bitbake world"
285         """
286         all_depends = self.status.all_depends
287         pn_provides = self.status.pn_provides
288         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
289         for f in self.status.possible_world:
290             terminal = True
291             pn = self.status.pkg_fn[f]
292
293             for p in pn_provides[pn]:
294                 if p.startswith('virtual/'):
295                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
296                     terminal = False
297                     break
298                 for pf in self.status.providers[p]:
299                     if self.status.pkg_fn[pf] != pn:
300                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
301                         terminal = False
302                         break
303             if terminal:
304                 self.status.world_target.add(pn)
305
306             # drop reference count now
307             self.status.possible_world = None
308             self.status.all_depends    = None
309
310     def myProgressCallback( self, x, y, f, from_cache ):
311         """Update any tty with the progress change"""
312         if os.isatty(sys.stdout.fileno()):
313             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
314             sys.stdout.flush()
315         else:
316             if x == 1:
317                 sys.stdout.write("Parsing .bb files, please wait...")
318                 sys.stdout.flush()
319             if x == y:
320                 sys.stdout.write("done.")
321                 sys.stdout.flush()
322
323     def interactiveMode( self ):
324         """Drop off into a shell"""
325         try:
326             from bb import shell
327         except ImportError, details:
328             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
329         else:
330             bb.data.update_data( self.configuration.data )
331             bb.data.expandKeys( self.configuration.data )
332             shell.start( self )
333             sys.exit( 0 )
334
335     def parseConfigurationFile( self, afile ):
336         try:
337             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
338
339             # Add the handlers we inherited by INHERIT
340             # we need to do this manually as it is not guranteed
341             # we will pick up these classes... as we only INHERIT
342             # on .inc and .bb files but not on .conf
343             data = bb.data.createCopy( self.configuration.data )
344             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
345             for inherit in inherits:
346                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
347
348             # FIXME: This assumes that we included at least one .inc file
349             for var in bb.data.keys(data):
350                 if bb.data.getVarFlag(var, 'handler', data):
351                     bb.event.register(var,bb.data.getVar(var, data))
352
353             bb.fetch.fetcher_init(self.configuration.data)
354
355             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
356
357         except IOError:
358             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
359         except bb.parse.ParseError, details:
360             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
361
362     def handleCollections( self, collections ):
363         """Handle collections"""
364         if collections:
365             collection_list = collections.split()
366             for c in collection_list:
367                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
368                 if regex == None:
369                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
370                     continue
371                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
372                 if priority == None:
373                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
374                     continue
375                 try:
376                     cre = re.compile(regex)
377                 except re.error:
378                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
379                     continue
380                 try:
381                     pri = int(priority)
382                     self.status.bbfile_config_priorities.append((cre, pri))
383                 except ValueError:
384                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
385
386     def buildSetVars(self):
387         """
388         Setup any variables needed before starting a build
389         """
390         if not bb.data.getVar("BUILDNAME", self.configuration.data):
391             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
392         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
393
394     def matchFile(self, buildfile):
395         """
396         Convert the fragment buildfile into a real file
397         Error if there are too many matches
398         """
399         bf = os.path.abspath(buildfile)
400         try:
401             os.stat(bf)
402             return bf
403         except OSError:
404             (filelist, masked) = self.collect_bbfiles()
405             regexp = re.compile(buildfile)
406             matches = []
407             for f in filelist:
408                 if regexp.search(f) and os.path.isfile(f):
409                     bf = f
410                     matches.append(f)
411             if len(matches) != 1:
412                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
413                 for f in matches:
414                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
415                 sys.exit(1)
416             return matches[0]               
417
418     def buildFile(self, buildfile):
419         """
420         Build the file matching regexp buildfile
421         """
422
423         bf = self.matchFile(buildfile)
424
425         bbfile_data = bb.parse.handle(bf, self.configuration.data)
426
427         # Remove stamp for target if force mode active
428         if self.configuration.force:
429             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
430             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
431
432         item = bb.data.getVar('PN', bbfile_data, 1)
433         try:
434             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
435         except bb.build.EventException:
436             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
437
438         sys.exit(0)
439
440     def buildTargets(self, targets):
441         """
442         Attempt to build the targets specified
443         """
444
445         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
446         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
447
448         localdata = data.createCopy(self.configuration.data)
449         bb.data.update_data(localdata)
450         bb.data.expandKeys(localdata)
451
452         taskdata = bb.taskdata.TaskData(self.configuration.abort)
453
454         runlist = []
455         try:
456             for k in targets:
457                 taskdata.add_provider(localdata, self.status, k)
458                 runlist.append([k, "do_%s" % self.configuration.cmd])
459             taskdata.add_unresolved(localdata, self.status)
460         except bb.providers.NoProvider:
461             sys.exit(1)
462
463         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
464         rq.prepare_runqueue()
465         try:
466             failures = rq.execute_runqueue()
467         except runqueue.TaskFailure, fnids:
468             for fnid in fnids:
469                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
470             sys.exit(1)
471         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
472
473         sys.exit(0)
474
475     def updateCache(self):
476         # Import Psyco if available and not disabled
477         if not self.configuration.disable_psyco:
478             try:
479                 import psyco
480             except ImportError:
481                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
482             else:
483                 psyco.bind( self.parse_bbfiles )
484         else:
485             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
486
487         self.status = bb.cache.CacheData()
488
489         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
490         self.status.ignored_dependencies = Set( ignore.split() )
491
492         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
493
494         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
495         (filelist, masked) = self.collect_bbfiles()
496         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
497         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
498
499         self.buildDepgraph()
500
501     def cook(self):
502         """
503         We are building stuff here. We do the building
504         from here. By default we try to execute task
505         build.
506         """
507
508         if self.configuration.show_environment:
509             self.showEnvironment()
510             sys.exit( 0 )
511
512         self.buildSetVars()
513
514         if self.configuration.interactive:
515             self.interactiveMode()
516
517         if self.configuration.buildfile is not None:
518             return self.buildFile(self.configuration.buildfile)
519
520         # initialise the parsing status now we know we will need deps
521         self.updateCache()
522
523         if self.configuration.parse_only:
524             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
525             return 0
526
527         pkgs_to_build = self.configuration.pkgs_to_build
528
529         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
530         if bbpkgs:
531             pkgs_to_build.extend(bbpkgs.split())
532         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
533                              and not self.configuration.show_environment:
534                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
535                 print "for usage information."
536                 sys.exit(0)
537
538         try:
539             if self.configuration.show_versions:
540                 self.showVersions()
541                 sys.exit( 0 )
542             if 'world' in pkgs_to_build:
543                 self.buildWorldTargetList()
544                 pkgs_to_build.remove('world')
545                 for t in self.status.world_target:
546                     pkgs_to_build.append(t)
547
548             if self.configuration.dot_graph:
549                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
550                 sys.exit( 0 )
551
552             return self.buildTargets(pkgs_to_build)
553
554         except KeyboardInterrupt:
555             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
556             sys.exit(1)
557
558     def get_bbfiles( self, path = os.getcwd() ):
559         """Get list of default .bb files by reading out the current directory"""
560         contents = os.listdir(path)
561         bbfiles = []
562         for f in contents:
563             (root, ext) = os.path.splitext(f)
564             if ext == ".bb":
565                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
566         return bbfiles
567
568     def find_bbfiles( self, path ):
569         """Find all the .bb files in a directory"""
570         from os.path import join
571
572         found = []
573         for dir, dirs, files in os.walk(path):
574             for ignored in ('SCCS', 'CVS', '.svn'):
575                 if ignored in dirs:
576                     dirs.remove(ignored)
577             found += [join(dir,f) for f in files if f.endswith('.bb')]
578
579         return found
580
581     def collect_bbfiles( self ):
582         """Collect all available .bb build files"""
583         parsed, cached, skipped, masked = 0, 0, 0, 0
584         self.bb_cache = bb.cache.init(self)
585
586         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
587         data.setVar("BBFILES", " ".join(files), self.configuration.data)
588
589         if not len(files):
590             files = self.get_bbfiles()
591
592         if not len(files):
593             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
594
595         newfiles = []
596         for f in files:
597             if os.path.isdir(f):
598                 dirfiles = self.find_bbfiles(f)
599                 if dirfiles:
600                     newfiles += dirfiles
601                     continue
602             newfiles += glob.glob(f) or [ f ]
603
604         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
605
606         if not bbmask:
607             return (newfiles, 0)
608
609         try:
610             bbmask_compiled = re.compile(bbmask)
611         except sre_constants.error:
612             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
613
614         finalfiles = []
615         for i in xrange( len( newfiles ) ):
616             f = newfiles[i]
617             if bbmask and bbmask_compiled.search(f):
618                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
619                 masked += 1
620                 continue
621             finalfiles.append(f)
622
623         return (finalfiles, masked)
624
625     def parse_bbfiles(self, filelist, masked, progressCallback = None):
626         parsed, cached, skipped, error = 0, 0, 0, 0
627         for i in xrange( len( filelist ) ):
628             f = filelist[i]
629
630             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
631
632             # read a file's metadata
633             try:
634                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
635                 if skip:
636                     skipped += 1
637                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
638                     self.bb_cache.skip(f)
639                     continue
640                 elif fromCache: cached += 1
641                 else: parsed += 1
642                 deps = None
643
644                 # Disabled by RP as was no longer functional
645                 # allow metadata files to add items to BBFILES
646                 #data.update_data(self.pkgdata[f])
647                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
648                 #if addbbfiles:
649                 #    for aof in addbbfiles.split():
650                 #        if not files.count(aof):
651                 #            if not os.path.isabs(aof):
652                 #                aof = os.path.join(os.path.dirname(f),aof)
653                 #            files.append(aof)
654
655                 self.bb_cache.handle_data(f, self.status)
656
657                 # now inform the caller
658                 if progressCallback is not None:
659                     progressCallback( i + 1, len( filelist ), f, fromCache )
660
661             except IOError, e:
662                 self.bb_cache.remove(f)
663                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
664                 pass
665             except KeyboardInterrupt:
666                 self.bb_cache.sync()
667                 raise
668             except Exception, e:
669                 error += 1
670                 self.bb_cache.remove(f)
671                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
672             except:
673                 self.bb_cache.remove(f)
674                 raise
675
676         if progressCallback is not None:
677             print "\r" # need newline after Handling Bitbake files message
678             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
679
680         self.bb_cache.sync()
681
682         if error > 0:
683             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")