bitbake: use Psyco only on x86 (no support for other archs) - taken from Poky
[bitbake.git] / lib / bb / cooker.py
1 #!/usr/bin/env python
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4 #
5 # Copyright (C) 2003, 2004  Chris Larson
6 # Copyright (C) 2003, 2004  Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005        Holger Hans Peter Freyther
9 # Copyright (C) 2005        ROAD GmbH
10 # Copyright (C) 2006        Richard Purdie
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License version 2 as
14 # published by the Free Software Foundation.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License along
22 # with this program; if not, write to the Free Software Foundation, Inc.,
23 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25 import sys, os, getopt, glob, copy, os.path, re, time
26 import bb
27 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
28 from sets import Set
29 import itertools, sre_constants
30
31 parsespin = itertools.cycle( r'|/-\\' )
32
33 #============================================================================#
34 # BBCooker
35 #============================================================================#
36 class BBCooker:
37     """
38     Manages one bitbake build run
39     """
40
41     def __init__(self, configuration):
42         self.status = None
43
44         self.cache = None
45         self.bb_cache = None
46
47         self.configuration = configuration
48
49         if self.configuration.verbose:
50             bb.msg.set_verbose(True)
51
52         if self.configuration.debug:
53             bb.msg.set_debug_level(self.configuration.debug)
54         else:
55             bb.msg.set_debug_level(0)
56
57         if self.configuration.debug_domains:
58             bb.msg.set_debug_domains(self.configuration.debug_domains)
59
60         self.configuration.data = bb.data.init()
61
62         for f in self.configuration.file:
63             self.parseConfigurationFile( f )
64
65         self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
66
67         if not self.configuration.cmd:
68             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build"
69
70         #
71         # Special updated configuration we use for firing events
72         #
73         self.configuration.event_data = bb.data.createCopy(self.configuration.data)
74         bb.data.update_data(self.configuration.event_data)
75
76         #
77         # TOSTOP must not be set or our children will hang when they output
78         #
79         fd = sys.stdout.fileno()
80         if os.isatty(fd):
81             import termios
82             tcattr = termios.tcgetattr(fd)
83             if tcattr[3] & termios.TOSTOP:
84                 bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
85                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
86                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
87
88
89     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
90         """
91         Build one task of a package, optionally build following task depends
92         """
93         bb.event.fire(bb.event.PkgStarted(item, the_data))
94         try:
95             if not build_depends:
96                 bb.data.setVarFlag('do_%s' % task, 'dontrundeps', 1, the_data)
97             if not self.configuration.dry_run:
98                 bb.build.exec_task('do_%s' % task, the_data)
99             bb.event.fire(bb.event.PkgSucceeded(item, the_data))
100             return True
101         except bb.build.FuncFailed:
102             bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
103             bb.event.fire(bb.event.PkgFailed(item, the_data))
104             raise
105         except bb.build.EventException, e:
106             event = e.args[1]
107             bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
108             bb.event.fire(bb.event.PkgFailed(item, the_data))
109             raise
110
111     def tryBuild( self, fn, build_depends):
112         """
113         Build a provider and its dependencies. 
114         build_depends is a list of previous build dependencies (not runtime)
115         If build_depends is empty, we're dealing with a runtime depends
116         """
117
118         the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
119
120         item = self.status.pkg_fn[fn]
121
122         if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
123             return True
124
125         return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data, build_depends)
126
127     def showVersions(self):
128         pkg_pn = self.status.pkg_pn
129         preferred_versions = {}
130         latest_versions = {}
131
132         # Sort by priority
133         for pn in pkg_pn.keys():
134             (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
135             preferred_versions[pn] = (pref_ver, pref_file)
136             latest_versions[pn] = (last_ver, last_file)
137
138         pkg_list = pkg_pn.keys()
139         pkg_list.sort()
140
141         for p in pkg_list:
142             pref = preferred_versions[p]
143             latest = latest_versions[p]
144
145             if pref != latest:
146                 prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
147             else:
148                 prefstr = ""
149
150             print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2],
151                                         prefstr)
152
153
154     def showEnvironment( self ):
155         """Show the outer or per-package environment"""
156         if self.configuration.buildfile:
157             self.cb = None
158             self.bb_cache = bb.cache.init(self)
159             bf = self.matchFile(self.configuration.buildfile)
160             try:
161                 self.configuration.data = self.bb_cache.loadDataFull(bf, self.configuration.data)
162             except IOError, e:
163                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (bf, e))
164             except Exception, e:
165                 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
166         # emit variables and shell functions
167         try:
168             data.update_data( self.configuration.data )
169             data.emit_env(sys.__stdout__, self.configuration.data, True)
170         except Exception, e:
171             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
172         # emit the metadata which isnt valid shell
173         data.expandKeys( self.configuration.data )      
174         for e in self.configuration.data.keys():
175             if data.getVarFlag( e, 'python', self.configuration.data ):
176                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
177
178     def generateDotGraph( self, pkgs_to_build, ignore_deps ):
179         """
180         Generate a task dependency graph. 
181
182         pkgs_to_build A list of packages that needs to be built
183         ignore_deps   A list of names where processing of dependencies
184                       should be stopped. e.g. dependencies that get
185         """
186
187         for dep in ignore_deps:
188             self.status.ignored_dependencies.add(dep)
189
190         localdata = data.createCopy(self.configuration.data)
191         bb.data.update_data(localdata)
192         bb.data.expandKeys(localdata)
193         taskdata = bb.taskdata.TaskData(self.configuration.abort)
194
195         runlist = []
196         try:
197             for k in pkgs_to_build:
198                 taskdata.add_provider(localdata, self.status, k)
199                 runlist.append([k, "do_%s" % self.configuration.cmd])
200             taskdata.add_unresolved(localdata, self.status)
201         except bb.providers.NoProvider:
202             sys.exit(1)
203         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
204         rq.prepare_runqueue()
205
206         seen_fnids = []  
207         depends_file = file('depends.dot', 'w' )
208         tdepends_file = file('task-depends.dot', 'w' )
209         print >> depends_file, "digraph depends {"
210         print >> tdepends_file, "digraph depends {"
211         rq.prio_map.reverse()
212         for task1 in range(len(rq.runq_fnid)):
213             task = rq.prio_map[task1]
214             taskname = rq.runq_task[task]
215             fnid = rq.runq_fnid[task]
216             fn = taskdata.fn_index[fnid]
217             pn = self.status.pkg_fn[fn]
218             version  = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
219             print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn)
220             for dep in rq.runq_depends[task]:
221                 depfn = taskdata.fn_index[rq.runq_fnid[dep]]
222                 deppn = self.status.pkg_fn[depfn]
223                 print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep])
224             if fnid not in seen_fnids:
225                 seen_fnids.append(fnid)
226                 packages = []
227                 print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn)              
228                 for depend in self.status.deps[fn]:
229                     print >> depends_file, '"%s" -> "%s"' % (pn, depend)
230                 rdepends = self.status.rundeps[fn]
231                 for package in rdepends:
232                     for rdepend in rdepends[package]:
233                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
234                     packages.append(package)
235                 rrecs = self.status.runrecs[fn]
236                 for package in rrecs:
237                     for rdepend in rrecs[package]:
238                         print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend)
239                     if not package in packages:
240                         packages.append(package)
241                 for package in packages:
242                     if package != pn:
243                         print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn)
244                         for depend in self.status.deps[fn]:
245                             print >> depends_file, '"%s" -> "%s"' % (package, depend)
246                 # Prints a flattened form of the above where subpackages of a package are merged into the main pn
247                 #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn)
248                 #for rdep in taskdata.rdepids[fnid]:
249                 #    print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep])
250                 #for dep in taskdata.depids[fnid]:
251                 #    print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep])
252         print >> depends_file,  "}"
253         print >> tdepends_file,  "}"
254         bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'")
255         bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'")
256
257     def buildDepgraph( self ):
258         all_depends = self.status.all_depends
259         pn_provides = self.status.pn_provides
260
261         localdata = data.createCopy(self.configuration.data)
262         bb.data.update_data(localdata)
263         bb.data.expandKeys(localdata)
264
265         def calc_bbfile_priority(filename):
266             for (regex, pri) in self.status.bbfile_config_priorities:
267                 if regex.match(filename):
268                     return pri
269             return 0
270
271         # Handle PREFERRED_PROVIDERS
272         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
273             (providee, provider) = p.split(':')
274             if providee in self.status.preferred and self.status.preferred[providee] != provider:
275                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
276             self.status.preferred[providee] = provider
277
278         # Calculate priorities for each file
279         for p in self.status.pkg_fn.keys():
280             self.status.bbfile_priority[p] = calc_bbfile_priority(p)
281
282     def buildWorldTargetList(self):
283         """
284          Build package list for "bitbake world"
285         """
286         all_depends = self.status.all_depends
287         pn_provides = self.status.pn_provides
288         bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
289         for f in self.status.possible_world:
290             terminal = True
291             pn = self.status.pkg_fn[f]
292
293             for p in pn_provides[pn]:
294                 if p.startswith('virtual/'):
295                     bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
296                     terminal = False
297                     break
298                 for pf in self.status.providers[p]:
299                     if self.status.pkg_fn[pf] != pn:
300                         bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
301                         terminal = False
302                         break
303             if terminal:
304                 self.status.world_target.add(pn)
305
306             # drop reference count now
307             self.status.possible_world = None
308             self.status.all_depends    = None
309
310     def myProgressCallback( self, x, y, f, from_cache ):
311         """Update any tty with the progress change"""
312         if os.isatty(sys.stdout.fileno()):
313             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
314             sys.stdout.flush()
315         else:
316             if x == 1:
317                 sys.stdout.write("Parsing .bb files, please wait...")
318                 sys.stdout.flush()
319             if x == y:
320                 sys.stdout.write("done.")
321                 sys.stdout.flush()
322
323     def interactiveMode( self ):
324         """Drop off into a shell"""
325         try:
326             from bb import shell
327         except ImportError, details:
328             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
329         else:
330             bb.data.update_data( self.configuration.data )
331             bb.data.expandKeys( self.configuration.data )
332             shell.start( self )
333             sys.exit( 0 )
334
335     def parseConfigurationFile( self, afile ):
336         try:
337             self.configuration.data = bb.parse.handle( afile, self.configuration.data )
338
339             # Add the handlers we inherited by INHERIT
340             # we need to do this manually as it is not guranteed
341             # we will pick up these classes... as we only INHERIT
342             # on .inc and .bb files but not on .conf
343             data = bb.data.createCopy( self.configuration.data )
344             inherits  = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
345             for inherit in inherits:
346                 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
347
348             # FIXME: This assumes that we included at least one .inc file
349             for var in bb.data.keys(data):
350                 if bb.data.getVarFlag(var, 'handler', data):
351                     bb.event.register(var,bb.data.getVar(var, data))
352
353             bb.fetch.fetcher_init(self.configuration.data)
354
355             bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
356
357         except IOError:
358             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
359         except bb.parse.ParseError, details:
360             bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
361
362     def handleCollections( self, collections ):
363         """Handle collections"""
364         if collections:
365             collection_list = collections.split()
366             for c in collection_list:
367                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
368                 if regex == None:
369                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
370                     continue
371                 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
372                 if priority == None:
373                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
374                     continue
375                 try:
376                     cre = re.compile(regex)
377                 except re.error:
378                     bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
379                     continue
380                 try:
381                     pri = int(priority)
382                     self.status.bbfile_config_priorities.append((cre, pri))
383                 except ValueError:
384                     bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
385
386     def buildSetVars(self):
387         """
388         Setup any variables needed before starting a build
389         """
390         if not bb.data.getVar("BUILDNAME", self.configuration.data):
391             bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
392         bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
393
394     def matchFile(self, buildfile):
395         """
396         Convert the fragment buildfile into a real file
397         Error if there are too many matches
398         """
399         bf = os.path.abspath(buildfile)
400         try:
401             os.stat(bf)
402             return bf
403         except OSError:
404             (filelist, masked) = self.collect_bbfiles()
405             regexp = re.compile(buildfile)
406             matches = []
407             for f in filelist:
408                 if regexp.search(f) and os.path.isfile(f):
409                     bf = f
410                     matches.append(f)
411             if len(matches) != 1:
412                 bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
413                 for f in matches:
414                     bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
415                 sys.exit(1)
416             return matches[0]               
417
418     def buildFile(self, buildfile):
419         """
420         Build the file matching regexp buildfile
421         """
422
423         bf = self.matchFile(buildfile)
424
425         bbfile_data = bb.parse.handle(bf, self.configuration.data)
426
427         # Remove stamp for target if force mode active
428         if self.configuration.force:
429             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, bf))
430             bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
431
432         item = bb.data.getVar('PN', bbfile_data, 1)
433         try:
434             self.tryBuildPackage(bf, item, self.configuration.cmd, bbfile_data, True)
435         except bb.build.EventException:
436             bb.msg.error(bb.msg.domain.Build,  "Build of '%s' failed" % item )
437
438         sys.exit(0)
439
440     def buildTargets(self, targets):
441         """
442         Attempt to build the targets specified
443         """
444
445         buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
446         bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data))
447
448         localdata = data.createCopy(self.configuration.data)
449         bb.data.update_data(localdata)
450         bb.data.expandKeys(localdata)
451
452         taskdata = bb.taskdata.TaskData(self.configuration.abort)
453
454         runlist = []
455         try:
456             for k in targets:
457                 taskdata.add_provider(localdata, self.status, k)
458                 runlist.append([k, "do_%s" % self.configuration.cmd])
459             taskdata.add_unresolved(localdata, self.status)
460         except bb.providers.NoProvider:
461             sys.exit(1)
462
463         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
464         rq.prepare_runqueue()
465         try:
466             failures = rq.execute_runqueue()
467         except runqueue.TaskFailure, fnids:
468             for fnid in fnids:
469                 bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
470             sys.exit(1)
471         bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
472
473         sys.exit(0)
474
475     def updateCache(self):
476         # Import Psyco if available and not disabled
477         import platform
478         if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
479             if not self.configuration.disable_psyco:
480                 try:
481                     import psyco
482                 except ImportError:
483                     bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
484                 else:
485                     psyco.bind( self.parse_bbfiles )
486             else:
487                 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
488
489         self.status = bb.cache.CacheData()
490
491         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
492         self.status.ignored_dependencies = Set( ignore.split() )
493
494         self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
495
496         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
497         (filelist, masked) = self.collect_bbfiles()
498         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
499         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
500
501         self.buildDepgraph()
502
503     def cook(self):
504         """
505         We are building stuff here. We do the building
506         from here. By default we try to execute task
507         build.
508         """
509
510         if self.configuration.show_environment:
511             self.showEnvironment()
512             sys.exit( 0 )
513
514         self.buildSetVars()
515
516         if self.configuration.interactive:
517             self.interactiveMode()
518
519         if self.configuration.buildfile is not None:
520             return self.buildFile(self.configuration.buildfile)
521
522         # initialise the parsing status now we know we will need deps
523         self.updateCache()
524
525         if self.configuration.parse_only:
526             bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only.  Exiting.")
527             return 0
528
529         pkgs_to_build = self.configuration.pkgs_to_build
530
531         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
532         if bbpkgs:
533             pkgs_to_build.extend(bbpkgs.split())
534         if len(pkgs_to_build) == 0 and not self.configuration.show_versions \
535                              and not self.configuration.show_environment:
536                 print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
537                 print "for usage information."
538                 sys.exit(0)
539
540         try:
541             if self.configuration.show_versions:
542                 self.showVersions()
543                 sys.exit( 0 )
544             if 'world' in pkgs_to_build:
545                 self.buildWorldTargetList()
546                 pkgs_to_build.remove('world')
547                 for t in self.status.world_target:
548                     pkgs_to_build.append(t)
549
550             if self.configuration.dot_graph:
551                 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
552                 sys.exit( 0 )
553
554             return self.buildTargets(pkgs_to_build)
555
556         except KeyboardInterrupt:
557             bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
558             sys.exit(1)
559
560     def get_bbfiles( self, path = os.getcwd() ):
561         """Get list of default .bb files by reading out the current directory"""
562         contents = os.listdir(path)
563         bbfiles = []
564         for f in contents:
565             (root, ext) = os.path.splitext(f)
566             if ext == ".bb":
567                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
568         return bbfiles
569
570     def find_bbfiles( self, path ):
571         """Find all the .bb files in a directory"""
572         from os.path import join
573
574         found = []
575         for dir, dirs, files in os.walk(path):
576             for ignored in ('SCCS', 'CVS', '.svn'):
577                 if ignored in dirs:
578                     dirs.remove(ignored)
579             found += [join(dir,f) for f in files if f.endswith('.bb')]
580
581         return found
582
583     def collect_bbfiles( self ):
584         """Collect all available .bb build files"""
585         parsed, cached, skipped, masked = 0, 0, 0, 0
586         self.bb_cache = bb.cache.init(self)
587
588         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
589         data.setVar("BBFILES", " ".join(files), self.configuration.data)
590
591         if not len(files):
592             files = self.get_bbfiles()
593
594         if not len(files):
595             bb.msg.error(bb.msg.domain.Collection, "no files to build.")
596
597         newfiles = []
598         for f in files:
599             if os.path.isdir(f):
600                 dirfiles = self.find_bbfiles(f)
601                 if dirfiles:
602                     newfiles += dirfiles
603                     continue
604             newfiles += glob.glob(f) or [ f ]
605
606         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
607
608         if not bbmask:
609             return (newfiles, 0)
610
611         try:
612             bbmask_compiled = re.compile(bbmask)
613         except sre_constants.error:
614             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
615
616         finalfiles = []
617         for i in xrange( len( newfiles ) ):
618             f = newfiles[i]
619             if bbmask and bbmask_compiled.search(f):
620                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
621                 masked += 1
622                 continue
623             finalfiles.append(f)
624
625         return (finalfiles, masked)
626
627     def parse_bbfiles(self, filelist, masked, progressCallback = None):
628         parsed, cached, skipped, error = 0, 0, 0, 0
629         for i in xrange( len( filelist ) ):
630             f = filelist[i]
631
632             bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
633
634             # read a file's metadata
635             try:
636                 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
637                 if skip:
638                     skipped += 1
639                     bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
640                     self.bb_cache.skip(f)
641                     continue
642                 elif fromCache: cached += 1
643                 else: parsed += 1
644                 deps = None
645
646                 # Disabled by RP as was no longer functional
647                 # allow metadata files to add items to BBFILES
648                 #data.update_data(self.pkgdata[f])
649                 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
650                 #if addbbfiles:
651                 #    for aof in addbbfiles.split():
652                 #        if not files.count(aof):
653                 #            if not os.path.isabs(aof):
654                 #                aof = os.path.join(os.path.dirname(f),aof)
655                 #            files.append(aof)
656
657                 self.bb_cache.handle_data(f, self.status)
658
659                 # now inform the caller
660                 if progressCallback is not None:
661                     progressCallback( i + 1, len( filelist ), f, fromCache )
662
663             except IOError, e:
664                 self.bb_cache.remove(f)
665                 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
666                 pass
667             except KeyboardInterrupt:
668                 self.bb_cache.sync()
669                 raise
670             except Exception, e:
671                 error += 1
672                 self.bb_cache.remove(f)
673                 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
674             except:
675                 self.bb_cache.remove(f)
676                 raise
677
678         if progressCallback is not None:
679             print "\r" # need newline after Handling Bitbake files message
680             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
681
682         self.bb_cache.sync()
683
684         if error > 0:
685             bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...")